Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21    de::DeserializeOwned,
22    ser::{SerializeMap, SerializeSeq},
23    Deserialize, Deserializer, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
33/// Worker-registration capability for server-routed read-only queries.
34pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
35/// Worker-registration capability for synchronous workflow updates.
36pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
37/// First additive worker protocol that defines query-task transport.
38pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
39
40const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
41const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
42    "Workflow task waiting for scheduled history.";
43const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
44const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
45const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
46const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
47
48const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
49    "lease_expired",
50    "query_task_not_found",
51    "query_task_not_leased",
52    "query_task_timed_out",
53];
54
55/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
56pub const AVRO_VALUE_SCHEMA_JSON: &str =
57    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
58pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
59pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
60const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
61
62static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
63
64#[derive(Clone, Copy)]
65enum RequestProtocol {
66    ControlPlane,
67    Worker(&'static str),
68}
69
70pub type Result<T> = std::result::Result<T, Error>;
71
72#[derive(Debug, Error)]
73pub enum Error {
74    #[error("transport error: {0}")]
75    Transport(#[from] reqwest::Error),
76    #[error(
77        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
78    )]
79    InvalidBaseUrl,
80    #[error("json error: {0}")]
81    Json(#[from] serde_json::Error),
82    #[error("http {status}: {body}")]
83    Http {
84        status: reqwest::StatusCode,
85        body: String,
86    },
87    #[error("codec error: {0}")]
88    Codec(String),
89    #[error(transparent)]
90    QueryFailed(QueryFailure),
91    #[error(transparent)]
92    Protocol(ProtocolFailure),
93    #[error(transparent)]
94    NonDeterministicReplay(ReplayFailure),
95    #[error(transparent)]
96    ChildWorkflowFailed(ChildWorkflowFailure),
97    #[error(transparent)]
98    ActivityFailed(ActivityFailure),
99    #[error(transparent)]
100    WorkflowCommandRejected(WorkflowCommandRejection),
101    #[error(transparent)]
102    WorkflowFailed(WorkflowTerminalOutcome),
103    #[error(transparent)]
104    WorkflowCancelled(WorkflowTerminalOutcome),
105    #[error(transparent)]
106    WorkflowTerminated(WorkflowTerminalOutcome),
107    #[error(transparent)]
108    WorkflowTimedOut(WorkflowTerminalOutcome),
109    #[error(transparent)]
110    ActivityTaskRejected(ActivityTaskRejection),
111    #[error("workflow handler {0:?} is not registered")]
112    WorkflowNotRegistered(String),
113    #[error("activity handler {0:?} is not registered")]
114    ActivityNotRegistered(String),
115    #[error("workflow future yielded without emitting a durable command")]
116    WorkflowYieldedWithoutCommand,
117    #[error("workflow state lock is poisoned")]
118    WorkflowStatePoisoned,
119    #[error("timer duration is too large for the worker protocol")]
120    TimerDurationOverflow,
121    #[error("operation timed out")]
122    Timeout,
123    #[error(
124        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
125    )]
126    MissingRoleCredentials {
127        role: &'static str,
128        opposite_role: &'static str,
129    },
130    #[error("worker loop error: {0}")]
131    WorkerLoop(String),
132    #[error(
133        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
134    )]
135    UnsupportedUpdateValidators { workflow_type: String },
136    #[error("{primary}; worker deregistration also failed: {deregistration}")]
137    WorkerShutdown {
138        primary: Box<Error>,
139        deregistration: Box<Error>,
140    },
141    #[error("invalid child workflow options: {0}")]
142    InvalidChildWorkflowOptions(String),
143    #[error(transparent)]
144    InvalidActivityOptions(ActivityOptionsError),
145    #[error(transparent)]
146    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
147    #[doc(hidden)]
148    #[error("workflow requested continue as new")]
149    ContinueAsNew(ContinueAsNewRequest),
150}
151
152/// The lifecycle command sent to a workflow execution.
153#[derive(Clone, Copy, Debug, PartialEq, Eq)]
154pub enum WorkflowCommandKind {
155    Cancel,
156    Terminate,
157}
158
159impl WorkflowCommandKind {
160    fn as_str(self) -> &'static str {
161        match self {
162            Self::Cancel => "cancel",
163            Self::Terminate => "terminate",
164        }
165    }
166}
167
168/// Optional structured fields for a cancellation or termination request.
169#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
170pub struct WorkflowCommandOptions {
171    #[serde(skip_serializing_if = "Option::is_none")]
172    pub reason: Option<String>,
173    #[serde(skip_serializing_if = "Option::is_none")]
174    pub request_id: Option<String>,
175}
176
177/// Server-enforced timeout policy for a workflow start.
178///
179/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
180/// only bounds how long the caller waits. A server deadline produces a terminal
181/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
182/// `run_timeout`.
183#[derive(Clone, Debug, PartialEq, Eq)]
184pub struct WorkflowStartOptions {
185    pub execution_timeout_seconds: u64,
186    pub run_timeout_seconds: u64,
187}
188
189impl Default for WorkflowStartOptions {
190    fn default() -> Self {
191        Self {
192            execution_timeout_seconds: 3600,
193            run_timeout_seconds: 600,
194        }
195    }
196}
197
198impl WorkflowStartOptions {
199    pub fn new() -> Self {
200        Self::default()
201    }
202
203    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
204        self.execution_timeout_seconds = seconds;
205        self
206    }
207
208    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
209        self.run_timeout_seconds = seconds;
210        self
211    }
212
213    fn validate(&self) -> Result<()> {
214        if self.execution_timeout_seconds == 0 {
215            return Err(Error::Codec(
216                "execution_timeout_seconds must be at least 1".to_string(),
217            ));
218        }
219        if self.run_timeout_seconds == 0 {
220            return Err(Error::Codec(
221                "run_timeout_seconds must be at least 1".to_string(),
222            ));
223        }
224        if self.run_timeout_seconds > self.execution_timeout_seconds {
225            return Err(Error::Codec(
226                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
227            ));
228        }
229
230        Ok(())
231    }
232}
233
234/// Optional routing overrides for a continue-as-new transition.
235///
236/// Omitted values retain the current workflow type and task queue. Server-owned
237/// instance metadata is not accepted here and is carried by the server.
238#[derive(Clone, Debug, Default, PartialEq, Eq)]
239pub struct ContinueAsNewOptions {
240    pub workflow_type: Option<String>,
241    pub task_queue: Option<String>,
242}
243
244impl ContinueAsNewOptions {
245    pub fn new() -> Self {
246        Self::default()
247    }
248
249    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
250        self.workflow_type = Some(workflow_type.into());
251        self
252    }
253
254    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
255        self.task_queue = Some(task_queue.into());
256        self
257    }
258
259    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
260        for (field, value) in [
261            ("workflow_type", self.workflow_type.as_deref()),
262            ("task_queue", self.task_queue.as_deref()),
263        ] {
264            if value.is_some_and(|value| value.trim().is_empty()) {
265                return Err(ContinueAsNewOptionsError {
266                    field,
267                    message: format!("{field} must not be empty"),
268                });
269            }
270        }
271        Ok(())
272    }
273}
274
275/// A stable validation error raised before a continue-as-new command is emitted.
276#[derive(Clone, Debug, Error, PartialEq, Eq)]
277#[error("invalid continue-as-new option {field}: {message}")]
278pub struct ContinueAsNewOptionsError {
279    pub field: &'static str,
280    pub message: String,
281}
282
283/// Public history-budget information attached to the current workflow task.
284#[derive(Clone, Debug, Default, PartialEq, Eq)]
285pub struct WorkflowHistoryBudget {
286    pub event_count: u64,
287    pub size_bytes: Option<u64>,
288    pub continue_as_new_recommended: bool,
289    pub pressure: Option<String>,
290}
291
292#[doc(hidden)]
293#[derive(Clone, Debug)]
294pub struct ContinueAsNewRequest {
295    arguments: AvroValue,
296    options: ContinueAsNewOptions,
297}
298
299impl WorkflowCommandOptions {
300    pub fn new() -> Self {
301        Self::default()
302    }
303
304    pub fn reason(mut self, reason: impl Into<String>) -> Self {
305        self.reason = Some(reason.into());
306        self
307    }
308
309    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
310        self.request_id = Some(request_id.into());
311        self
312    }
313}
314
315/// The accepted, machine-readable result of a lifecycle command.
316#[derive(Clone, Debug, PartialEq)]
317pub struct WorkflowCommandResult {
318    pub command: WorkflowCommandKind,
319    pub workflow_id: String,
320    pub run_id: Option<String>,
321    pub outcome: Option<String>,
322    pub reason: Option<String>,
323    pub command_status: Option<String>,
324    pub raw: Value,
325}
326
327/// A stable rejection returned by instance- or selected-run lifecycle commands.
328#[derive(Clone, Debug, Error)]
329#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
330pub struct WorkflowCommandRejection {
331    pub command: WorkflowCommandKind,
332    pub status: u16,
333    pub reason: String,
334    pub message: String,
335    pub workflow_id: String,
336    pub run_id: Option<String>,
337    pub target_scope: Option<String>,
338    pub body: Value,
339}
340
341/// Stable terminal categories returned by [`WorkflowHandle::result`].
342#[derive(Clone, Copy, Debug, PartialEq, Eq)]
343pub enum WorkflowTerminalKind {
344    Failed,
345    Cancelled,
346    Terminated,
347    TimedOut,
348}
349
350/// A typed terminal workflow outcome with durable identity and failure metadata.
351///
352/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
353/// of parsing its display representation. Fields remain `None` when an older
354/// server did not publish that metadata.
355#[derive(Clone, Debug, Error)]
356#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
357pub struct WorkflowTerminalOutcome {
358    pub kind: WorkflowTerminalKind,
359    pub workflow_id: String,
360    pub run_id: Option<String>,
361    pub reason: String,
362    pub failure_category: Option<String>,
363    pub failure_id: Option<String>,
364    pub exception_type: Option<String>,
365    pub exception_class: Option<String>,
366    pub non_retryable: Option<bool>,
367    pub message: Option<String>,
368    pub exception: Option<Value>,
369    pub raw: Value,
370}
371
372/// A worker-side activity settlement or heartbeat rejected by durable state.
373#[derive(Clone, Debug, Error)]
374#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
375pub struct ActivityTaskRejection {
376    pub operation: String,
377    pub status: u16,
378    pub reason: String,
379    pub task_id: String,
380    pub activity_attempt_id: String,
381    pub cancel_requested: bool,
382    pub can_continue: Option<bool>,
383    pub run_closed_reason: Option<String>,
384    pub body: Value,
385}
386
387/// Stable validation categories for [`ActivityOptions`].
388#[derive(Clone, Copy, Debug, PartialEq, Eq)]
389pub enum ActivityOptionsErrorKind {
390    EmptyTaskQueue,
391    EmptyRetryPolicy,
392    InvalidMaxAttempts,
393    BackoffWithoutRetryBudget,
394    TooManyBackoffIntervals,
395    InvalidBackoffCoefficient,
396    BackoffGenerationTooLarge,
397    BackoffOverflow,
398    EmptyNonRetryableErrorType,
399    TimeoutNotPositive,
400    TimeoutOverflow,
401    TimeoutOrder,
402}
403
404/// A machine-readable activity-options validation failure.
405#[derive(Clone, Debug, Error, PartialEq, Eq)]
406#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
407pub struct ActivityOptionsError {
408    pub kind: ActivityOptionsErrorKind,
409    pub field: Option<&'static str>,
410    pub message: String,
411}
412
413impl ActivityOptionsError {
414    fn new(
415        kind: ActivityOptionsErrorKind,
416        field: Option<&'static str>,
417        message: impl Into<String>,
418    ) -> Self {
419        Self {
420            kind,
421            field,
422            message: message.into(),
423        }
424    }
425}
426
427/// Stable terminal categories returned when an awaited activity does not succeed.
428#[derive(Clone, Copy, Debug, PartialEq, Eq)]
429pub enum ActivityFailureKind {
430    Failed,
431    Cancelled,
432    TimedOut,
433}
434
435/// A stable, machine-readable terminal activity failure.
436///
437/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
438/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
439#[derive(Clone, Debug, Error)]
440#[error("activity failed ({reason}): {message}")]
441pub struct ActivityFailure {
442    pub kind: ActivityFailureKind,
443    pub reason: String,
444    pub message: String,
445    pub activity_execution_id: Option<String>,
446    pub activity_attempt_id: Option<String>,
447    pub activity_type: Option<String>,
448    pub activity_class: Option<String>,
449    pub attempt_number: Option<u64>,
450    pub failure_id: Option<String>,
451    pub failure_category: Option<String>,
452    pub timeout_kind: Option<String>,
453    pub non_retryable: bool,
454    pub exception_type: Option<String>,
455    pub exception_class: Option<String>,
456    pub code: Option<Value>,
457    pub exception: Option<Value>,
458}
459
460/// Stable terminal categories returned when an awaited child does not succeed.
461#[derive(Clone, Copy, Debug, PartialEq, Eq)]
462pub enum ChildWorkflowFailureKind {
463    Failed,
464    Cancelled,
465    Terminated,
466}
467
468/// A stable, machine-readable child workflow failure delivered to its parent.
469///
470/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
471/// of parsing the display message. Child and parent identifiers retain the
472/// relationship recorded in durable history across worker restarts.
473#[derive(Clone, Debug, Error)]
474#[error("child workflow failed ({reason}): {message}")]
475pub struct ChildWorkflowFailure {
476    pub kind: ChildWorkflowFailureKind,
477    pub reason: String,
478    pub message: String,
479    pub parent_workflow_id: Option<String>,
480    pub parent_workflow_run_id: Option<String>,
481    pub child_workflow_id: Option<String>,
482    pub child_workflow_run_id: Option<String>,
483    pub child_workflow_type: Option<String>,
484    pub failure_id: Option<String>,
485    pub failure_category: Option<String>,
486    pub exception_type: Option<String>,
487    pub exception_class: Option<String>,
488    pub non_retryable: bool,
489    pub code: Option<Value>,
490    pub exception: Option<Value>,
491}
492
493/// The identity of one durable workflow execution.
494#[derive(Clone, Debug, PartialEq, Eq)]
495pub struct WorkflowIdentity {
496    pub workflow_id: Option<String>,
497    pub run_id: Option<String>,
498}
499
500/// A successful child result together with its durable parent-child identity.
501#[derive(Clone, Debug, PartialEq)]
502pub struct ChildWorkflowResult {
503    pub parent: WorkflowIdentity,
504    pub child: WorkflowIdentity,
505    pub child_workflow_type: Option<String>,
506    pub result: Value,
507}
508
509/// Lossless successful child result for fixed Avro Value workflows.
510#[derive(Clone, Debug, PartialEq)]
511pub struct ChildWorkflowAvroResult {
512    pub parent: WorkflowIdentity,
513    pub child: WorkflowIdentity,
514    pub child_workflow_type: Option<String>,
515    pub result: AvroValue,
516}
517
518/// Server behavior when a parent closes while its child is still open.
519#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
520pub enum ParentClosePolicy {
521    #[default]
522    Abandon,
523    RequestCancel,
524    Terminate,
525}
526
527impl ParentClosePolicy {
528    fn as_str(self) -> &'static str {
529        match self {
530            Self::Abandon => "abandon",
531            Self::RequestCancel => "request_cancel",
532            Self::Terminate => "terminate",
533        }
534    }
535}
536
537/// Durable retry policy for one child workflow invocation.
538#[derive(Clone, Debug, Default, PartialEq, Eq)]
539pub struct ChildWorkflowRetryPolicy {
540    pub max_attempts: Option<u32>,
541    pub backoff_seconds: Vec<u64>,
542    pub non_retryable_error_types: Vec<String>,
543}
544
545/// Options recorded with a child-workflow command.
546///
547/// The task queue is mandatory so routing is explicit and replay-stable.
548#[derive(Clone, Debug, PartialEq, Eq)]
549pub struct ChildWorkflowOptions {
550    pub task_queue: String,
551    pub parent_close_policy: ParentClosePolicy,
552    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
553    pub execution_timeout_seconds: Option<u64>,
554    pub run_timeout_seconds: Option<u64>,
555}
556
557impl ChildWorkflowOptions {
558    pub fn new(task_queue: impl Into<String>) -> Self {
559        Self {
560            task_queue: task_queue.into(),
561            parent_close_policy: ParentClosePolicy::Abandon,
562            retry_policy: None,
563            execution_timeout_seconds: None,
564            run_timeout_seconds: None,
565        }
566    }
567
568    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
569        self.parent_close_policy = policy;
570        self
571    }
572
573    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
574        self.retry_policy = Some(policy);
575        self
576    }
577
578    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
579        self.execution_timeout_seconds = Some(seconds);
580        self
581    }
582
583    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
584        self.run_timeout_seconds = Some(seconds);
585        self
586    }
587}
588
589/// Backoff intervals for one durable activity retry policy.
590#[derive(Clone, Debug, PartialEq, Eq)]
591pub enum ActivityBackoff {
592    /// Use these intervals between attempts. The server repeats the final
593    /// interval if the retry budget contains more attempts than entries.
594    Explicit(Vec<Duration>),
595    /// Generate one interval for every retry using integer exponential growth.
596    Exponential {
597        initial_interval: Duration,
598        coefficient: u32,
599        maximum_interval: Option<Duration>,
600    },
601}
602
603/// Durable server-side retry policy for one activity execution.
604#[derive(Clone, Debug, Default, PartialEq, Eq)]
605pub struct ActivityRetryPolicy {
606    pub max_attempts: Option<u32>,
607    pub backoff: Option<ActivityBackoff>,
608    pub non_retryable_error_types: Vec<String>,
609}
610
611impl ActivityRetryPolicy {
612    /// Start a policy with a finite attempt budget, including the first attempt.
613    pub fn new(max_attempts: u32) -> Self {
614        Self {
615            max_attempts: Some(max_attempts),
616            ..Self::default()
617        }
618    }
619
620    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
621        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
622        self
623    }
624
625    pub fn exponential_backoff(
626        mut self,
627        initial_interval: Duration,
628        coefficient: u32,
629        maximum_interval: Option<Duration>,
630    ) -> Self {
631        self.backoff = Some(ActivityBackoff::Exponential {
632            initial_interval,
633            coefficient,
634            maximum_interval,
635        });
636        self
637    }
638
639    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
640        self.non_retryable_error_types.push(error_type.into());
641        self
642    }
643
644    pub fn non_retryable_error_types(
645        mut self,
646        error_types: impl IntoIterator<Item = impl Into<String>>,
647    ) -> Self {
648        self.non_retryable_error_types
649            .extend(error_types.into_iter().map(Into::into));
650        self
651    }
652}
653
654/// Options recorded atomically on one deterministic `schedule_activity` command.
655///
656/// Durations are rounded up to whole seconds when encoded, so the server never
657/// receives a shorter timeout or backoff than the caller requested.
658#[derive(Clone, Debug, Default, PartialEq, Eq)]
659pub struct ActivityOptions {
660    pub task_queue: Option<String>,
661    pub retry_policy: Option<ActivityRetryPolicy>,
662    pub start_to_close_timeout: Option<Duration>,
663    pub schedule_to_start_timeout: Option<Duration>,
664    pub schedule_to_close_timeout: Option<Duration>,
665    pub heartbeat_timeout: Option<Duration>,
666}
667
668impl ActivityOptions {
669    pub fn new() -> Self {
670        Self::default()
671    }
672
673    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
674        self.task_queue = Some(task_queue.into());
675        self
676    }
677
678    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
679        self.retry_policy = Some(policy);
680        self
681    }
682
683    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
684        self.start_to_close_timeout = Some(timeout);
685        self
686    }
687
688    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
689        self.schedule_to_start_timeout = Some(timeout);
690        self
691    }
692
693    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
694        self.schedule_to_close_timeout = Some(timeout);
695        self
696    }
697
698    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
699        self.heartbeat_timeout = Some(timeout);
700        self
701    }
702
703    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
704        if self
705            .task_queue
706            .as_deref()
707            .is_some_and(|queue| queue.trim().is_empty())
708        {
709            return Err(ActivityOptionsError::new(
710                ActivityOptionsErrorKind::EmptyTaskQueue,
711                Some("task_queue"),
712                "task_queue must not be empty",
713            ));
714        }
715
716        for (field, value) in [
717            ("start_to_close_timeout", self.start_to_close_timeout),
718            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
719            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
720            ("heartbeat_timeout", self.heartbeat_timeout),
721        ] {
722            if value.is_some_and(|value| value.is_zero()) {
723                return Err(ActivityOptionsError::new(
724                    ActivityOptionsErrorKind::TimeoutNotPositive,
725                    Some(field),
726                    format!("{field} must be positive"),
727                ));
728            }
729        }
730
731        validate_timeout_order(
732            "heartbeat_timeout",
733            self.heartbeat_timeout,
734            "start_to_close_timeout",
735            self.start_to_close_timeout,
736        )?;
737        validate_timeout_order(
738            "start_to_close_timeout",
739            self.start_to_close_timeout,
740            "schedule_to_close_timeout",
741            self.schedule_to_close_timeout,
742        )?;
743        validate_timeout_order(
744            "schedule_to_start_timeout",
745            self.schedule_to_start_timeout,
746            "schedule_to_close_timeout",
747            self.schedule_to_close_timeout,
748        )?;
749
750        Ok(ValidatedActivityOptions {
751            task_queue: self.task_queue.clone(),
752            retry_policy: self
753                .retry_policy
754                .as_ref()
755                .map(validate_activity_retry_policy)
756                .transpose()?,
757            start_to_close_timeout: timeout_seconds(
758                "start_to_close_timeout",
759                self.start_to_close_timeout,
760            )?,
761            schedule_to_start_timeout: timeout_seconds(
762                "schedule_to_start_timeout",
763                self.schedule_to_start_timeout,
764            )?,
765            schedule_to_close_timeout: timeout_seconds(
766                "schedule_to_close_timeout",
767                self.schedule_to_close_timeout,
768            )?,
769            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
770        })
771    }
772}
773
774#[derive(Clone, Debug)]
775struct ValidatedActivityOptions {
776    task_queue: Option<String>,
777    retry_policy: Option<Value>,
778    start_to_close_timeout: Option<u64>,
779    schedule_to_start_timeout: Option<u64>,
780    schedule_to_close_timeout: Option<u64>,
781    heartbeat_timeout: Option<u64>,
782}
783
784fn validate_timeout_order(
785    smaller_name: &'static str,
786    smaller: Option<Duration>,
787    larger_name: &'static str,
788    larger: Option<Duration>,
789) -> std::result::Result<(), ActivityOptionsError> {
790    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
791        return Err(ActivityOptionsError::new(
792            ActivityOptionsErrorKind::TimeoutOrder,
793            Some(smaller_name),
794            format!("{smaller_name} must be <= {larger_name}"),
795        ));
796    }
797    Ok(())
798}
799
800fn timeout_seconds(
801    field: &'static str,
802    value: Option<Duration>,
803) -> std::result::Result<Option<u64>, ActivityOptionsError> {
804    value
805        .map(|value| {
806            activity_protocol_seconds(value).ok_or_else(|| {
807                ActivityOptionsError::new(
808                    ActivityOptionsErrorKind::TimeoutOverflow,
809                    Some(field),
810                    format!("{field} is too large for the worker protocol"),
811                )
812            })
813        })
814        .transpose()
815}
816
817fn duration_seconds_ceil(value: Duration) -> Option<u64> {
818    value
819        .as_secs()
820        .checked_add(u64::from(value.subsec_nanos() > 0))
821}
822
823fn activity_protocol_seconds(value: Duration) -> Option<u64> {
824    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
825}
826
827fn validate_activity_retry_policy(
828    policy: &ActivityRetryPolicy,
829) -> std::result::Result<Value, ActivityOptionsError> {
830    if policy.max_attempts.is_none()
831        && policy.backoff.is_none()
832        && policy.non_retryable_error_types.is_empty()
833    {
834        return Err(ActivityOptionsError::new(
835            ActivityOptionsErrorKind::EmptyRetryPolicy,
836            Some("retry_policy"),
837            "retry_policy must configure at least one field",
838        ));
839    }
840    if policy.max_attempts == Some(0) {
841        return Err(ActivityOptionsError::new(
842            ActivityOptionsErrorKind::InvalidMaxAttempts,
843            Some("retry_policy.max_attempts"),
844            "max_attempts must be >= 1",
845        ));
846    }
847    if policy
848        .non_retryable_error_types
849        .iter()
850        .any(|error_type| error_type.trim().is_empty())
851    {
852        return Err(ActivityOptionsError::new(
853            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
854            Some("retry_policy.non_retryable_error_types"),
855            "non_retryable_error_types must not contain empty values",
856        ));
857    }
858
859    let backoff_seconds = match &policy.backoff {
860        None => None,
861        Some(backoff) => {
862            let max_attempts = policy.max_attempts.ok_or_else(|| {
863                ActivityOptionsError::new(
864                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
865                    Some("retry_policy.backoff"),
866                    "backoff requires max_attempts",
867                )
868            })?;
869            let retry_count = max_attempts.saturating_sub(1) as usize;
870            let intervals = match backoff {
871                ActivityBackoff::Explicit(intervals) => {
872                    if intervals.len() > retry_count {
873                        return Err(ActivityOptionsError::new(
874                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
875                            Some("retry_policy.backoff"),
876                            "backoff interval count must not exceed max_attempts - 1",
877                        ));
878                    }
879                    intervals.clone()
880                }
881                ActivityBackoff::Exponential {
882                    initial_interval,
883                    coefficient,
884                    maximum_interval,
885                } => {
886                    if *coefficient < 1 {
887                        return Err(ActivityOptionsError::new(
888                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
889                            Some("retry_policy.backoff.coefficient"),
890                            "backoff coefficient must be >= 1",
891                        ));
892                    }
893                    if retry_count > 10_000 {
894                        return Err(ActivityOptionsError::new(
895                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
896                            Some("retry_policy.max_attempts"),
897                            "generated backoff supports at most 10000 retry intervals",
898                        ));
899                    }
900                    let mut current = *initial_interval;
901                    let mut intervals = Vec::with_capacity(retry_count);
902                    for _ in 0..retry_count {
903                        let interval = maximum_interval
904                            .map(|maximum| current.min(maximum))
905                            .unwrap_or(current);
906                        intervals.push(interval);
907                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
908                            break;
909                        }
910                        current = current.checked_mul(*coefficient).ok_or_else(|| {
911                            ActivityOptionsError::new(
912                                ActivityOptionsErrorKind::BackoffOverflow,
913                                Some("retry_policy.backoff"),
914                                "generated backoff interval overflowed",
915                            )
916                        })?;
917                    }
918                    intervals
919                }
920            };
921            Some(
922                intervals
923                    .into_iter()
924                    .map(|interval| {
925                        activity_protocol_seconds(interval).ok_or_else(|| {
926                            ActivityOptionsError::new(
927                                ActivityOptionsErrorKind::BackoffOverflow,
928                                Some("retry_policy.backoff"),
929                                "backoff interval is too large for the worker protocol",
930                            )
931                        })
932                    })
933                    .collect::<std::result::Result<Vec<_>, _>>()?,
934            )
935        }
936    };
937
938    let mut encoded = serde_json::Map::new();
939    if let Some(max_attempts) = policy.max_attempts {
940        encoded.insert("max_attempts".to_string(), json!(max_attempts));
941    }
942    if let Some(backoff_seconds) = backoff_seconds {
943        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
944    }
945    if !policy.non_retryable_error_types.is_empty() {
946        let mut canonical_error_types = Vec::new();
947        for error_type in policy
948            .non_retryable_error_types
949            .iter()
950            .map(|error_type| error_type.trim())
951        {
952            if !canonical_error_types.contains(&error_type) {
953                canonical_error_types.push(error_type);
954            }
955        }
956        encoded.insert(
957            "non_retryable_error_types".to_string(),
958            json!(canonical_error_types),
959        );
960    }
961    Ok(Value::Object(encoded))
962}
963
964/// A stable, machine-readable failure raised when workflow code no longer
965/// reconstructs the durable command stream recorded in history.
966#[derive(Clone, Debug, Error)]
967#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
968pub struct ReplayFailure {
969    pub reason: String,
970    pub sequence: Option<u64>,
971    pub expected: Option<String>,
972    pub actual: Option<String>,
973    pub message: String,
974}
975
976impl ReplayFailure {
977    fn new(
978        reason: impl Into<String>,
979        sequence: Option<u64>,
980        expected: Option<String>,
981        actual: Option<String>,
982        message: impl Into<String>,
983    ) -> Self {
984        Self {
985            reason: reason.into(),
986            sequence,
987            expected,
988            actual,
989            message: message.into(),
990        }
991    }
992}
993
994/// A stable, machine-readable workflow query or query-task settlement failure.
995#[derive(Clone, Debug, Error)]
996#[error("query failed ({reason}, HTTP {status}): {message}")]
997pub struct QueryFailure {
998    pub status: u16,
999    pub reason: String,
1000    pub message: String,
1001    pub body: Value,
1002}
1003
1004/// A stable failure returned when a server rejects an SDK protocol version.
1005#[derive(Clone, Debug, Error)]
1006#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1007pub struct ProtocolFailure {
1008    pub status: u16,
1009    pub reason: String,
1010    pub message: String,
1011    pub supported_version: Option<String>,
1012    pub requested_version: Option<String>,
1013    pub body: Value,
1014}
1015
1016#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1017pub struct PayloadEnvelope {
1018    pub codec: String,
1019    pub blob: String,
1020}
1021
1022impl PayloadEnvelope {
1023    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1024        encode_payload(value, DEFAULT_CODEC)
1025    }
1026
1027    /// Encode an explicit typed value, including the bytes branch that JSON
1028    /// serialization cannot represent.
1029    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1030        encode_avro_value(value)
1031    }
1032}
1033
1034/// Native adapter for the fixed language-neutral Avro Value schema.
1035#[derive(Clone, Debug, PartialEq)]
1036pub enum AvroValue {
1037    Null,
1038    Boolean(bool),
1039    Long(i64),
1040    Double(f64),
1041    Bytes(Vec<u8>),
1042    String(String),
1043    Array(Vec<AvroValue>),
1044    Map(BTreeMap<String, AvroValue>),
1045}
1046
1047impl AvroValue {
1048    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1049        Self::from_serde_value(
1050            serde_value::to_value(value).map_err(|error| {
1051                Error::Codec(format!("could not adapt value for Avro: {error}"))
1052            })?,
1053        )
1054    }
1055
1056    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1057        use serde_value::Value as SerdeValue;
1058
1059        match value {
1060            SerdeValue::Unit => Ok(Self::Null),
1061            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1062            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1063            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1064            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1065            SerdeValue::I64(value) => Ok(Self::Long(value)),
1066            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1067            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1068            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1069            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1070                Error::Codec(
1071                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1072                        .to_string(),
1073                )
1074            }),
1075            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1076            SerdeValue::F64(value) => Self::finite_double(value),
1077            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1078            SerdeValue::String(value) => Ok(Self::String(value)),
1079            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1080            SerdeValue::Option(None) => Ok(Self::Null),
1081            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1082                Self::from_serde_value(*value)
1083            }
1084            SerdeValue::Seq(values) => values
1085                .into_iter()
1086                .map(Self::from_serde_value)
1087                .collect::<Result<Vec<_>>>()
1088                .map(Self::Array),
1089            SerdeValue::Map(values) => values
1090                .into_iter()
1091                .map(|(key, value)| {
1092                    let SerdeValue::String(key) = key else {
1093                        return Err(Error::Codec(
1094                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1095                        ));
1096                    };
1097
1098                    Ok((key, Self::from_serde_value(value)?))
1099                })
1100                .collect::<Result<BTreeMap<_, _>>>()
1101                .map(Self::Map),
1102        }
1103    }
1104
1105    fn finite_double(value: f64) -> Result<Self> {
1106        if !value.is_finite() {
1107            return Err(Error::Codec(
1108                "non_finite_float: Avro Value doubles must be finite".to_string(),
1109            ));
1110        }
1111
1112        Ok(Self::Double(value))
1113    }
1114
1115    fn into_json(self) -> Result<Value> {
1116        match self {
1117            Self::Null => Ok(Value::Null),
1118            Self::Boolean(value) => Ok(Value::Bool(value)),
1119            Self::Long(value) => Ok(Value::Number(value.into())),
1120            Self::Double(value) => serde_json::Number::from_f64(value)
1121                .map(Value::Number)
1122                .ok_or_else(|| {
1123                    Error::Codec(
1124                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1125                    )
1126                }),
1127            Self::Bytes(value) => Ok(json!({
1128                "$type": "bytes",
1129                "base64": BASE64.encode(value),
1130            })),
1131            Self::String(value) => Ok(Value::String(value)),
1132            Self::Array(values) => values
1133                .into_iter()
1134                .map(Self::into_json)
1135                .collect::<Result<Vec<_>>>()
1136                .map(Value::Array),
1137            Self::Map(values) => values
1138                .into_iter()
1139                .map(|(key, value)| Ok((key, value.into_json()?)))
1140                .collect::<Result<serde_json::Map<_, _>>>()
1141                .map(Value::Object),
1142        }
1143    }
1144
1145    fn into_serde_value(self) -> serde_value::Value {
1146        use serde_value::Value as SerdeValue;
1147
1148        match self {
1149            Self::Null => SerdeValue::Unit,
1150            Self::Boolean(value) => SerdeValue::Bool(value),
1151            Self::Long(value) => SerdeValue::I64(value),
1152            Self::Double(value) => SerdeValue::F64(value),
1153            Self::Bytes(value) => SerdeValue::Bytes(value),
1154            Self::String(value) => SerdeValue::String(value),
1155            Self::Array(values) => {
1156                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1157            }
1158            Self::Map(values) => SerdeValue::Map(
1159                values
1160                    .into_iter()
1161                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1162                    .collect(),
1163            ),
1164        }
1165    }
1166
1167    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1168        self.into_serde_value().deserialize_into().map_err(|error| {
1169            Error::Codec(format!(
1170                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1171            ))
1172        })
1173    }
1174}
1175
1176impl Serialize for AvroValue {
1177    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1178    where
1179        S: Serializer,
1180    {
1181        match self {
1182            Self::Null => serializer.serialize_unit(),
1183            Self::Boolean(value) => serializer.serialize_bool(*value),
1184            Self::Long(value) => serializer.serialize_i64(*value),
1185            Self::Double(value) => serializer.serialize_f64(*value),
1186            Self::Bytes(value) => serializer.serialize_bytes(value),
1187            Self::String(value) => serializer.serialize_str(value),
1188            Self::Array(values) => {
1189                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1190                for value in values {
1191                    sequence.serialize_element(value)?;
1192                }
1193                sequence.end()
1194            }
1195            Self::Map(values) => {
1196                let mut map = serializer.serialize_map(Some(values.len()))?;
1197                for (key, value) in values {
1198                    map.serialize_entry(key, value)?;
1199                }
1200                map.end()
1201            }
1202        }
1203    }
1204}
1205
1206pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1207    let datum = avro_value_to_datum(value)?;
1208    let datum = to_avro_datum(avro_value_schema()?, datum)
1209        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1210    let mut bytes = Vec::with_capacity(datum.len() + 10);
1211    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1212    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1213    bytes.extend_from_slice(&datum);
1214    Ok(PayloadEnvelope {
1215        codec: DEFAULT_CODEC.to_string(),
1216        blob: BASE64.encode(bytes),
1217    })
1218}
1219
1220pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1221    if envelope.codec != DEFAULT_CODEC {
1222        return Err(unsupported_payload_codec(&envelope.codec));
1223    }
1224    decode_avro_value_blob(&envelope.blob)
1225}
1226
1227pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1228    let blob = match codec {
1229        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1230        other => return Err(unsupported_payload_codec(other)),
1231    };
1232
1233    Ok(PayloadEnvelope {
1234        codec: codec.to_string(),
1235        blob,
1236    })
1237}
1238
1239pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1240    match envelope.codec.as_str() {
1241        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1242        other => Err(unsupported_payload_codec(other)),
1243    }
1244}
1245
1246#[cfg(test)]
1247fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1248    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1249}
1250
1251fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1252    validate_payload_codec(fallback_codec)?;
1253
1254    if value.is_null() {
1255        return Ok(Value::Null);
1256    }
1257
1258    if let Some((codec, blob)) = payload_envelope_parts(value)? {
1259        return decode_blob(blob, codec);
1260    }
1261
1262    if let Some(blob) = value.as_str() {
1263        return decode_blob(blob, fallback_codec);
1264    }
1265
1266    Err(untagged_payload_value())
1267}
1268
1269fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1270    let envelope = match codec {
1271        DEFAULT_CODEC => encode_avro_value(value)?,
1272        other => return Err(unsupported_payload_codec(other)),
1273    };
1274    Ok(serde_json::to_value(envelope)?)
1275}
1276
1277fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1278    validate_payload_codec(fallback_codec)?;
1279
1280    if value.is_null() {
1281        return Ok(AvroValue::Null);
1282    }
1283
1284    if let Some((codec, blob)) = payload_envelope_parts(value)? {
1285        validate_payload_codec(codec)?;
1286        return decode_avro_value_blob(blob);
1287    }
1288
1289    if let Some(blob) = value.as_str() {
1290        return match fallback_codec {
1291            DEFAULT_CODEC => decode_avro_value_blob(blob),
1292            other => Err(unsupported_payload_codec(other)),
1293        };
1294    }
1295
1296    Err(untagged_payload_value())
1297}
1298
1299fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1300    match value {
1301        AvroValue::Null => AvroValue::Array(Vec::new()),
1302        AvroValue::Array(_) => value,
1303        other => AvroValue::Array(vec![other]),
1304    }
1305}
1306
1307fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1308    match codec {
1309        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1310        other => Err(unsupported_payload_codec(other)),
1311    }
1312}
1313
1314fn validate_payload_codec(codec: &str) -> Result<()> {
1315    match codec {
1316        DEFAULT_CODEC => Ok(()),
1317        MISSING_TASK_PAYLOAD_CODEC => {
1318            Err(invalid_task_payload_codec("task payload_codec is missing"))
1319        }
1320        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
1321        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
1322            "task payload_codec must be a string",
1323        )),
1324        other => Err(unsupported_payload_codec(other)),
1325    }
1326}
1327
1328fn invalid_task_payload_codec(reason: &str) -> Error {
1329    Error::Codec(format!(
1330        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
1331    ))
1332}
1333
1334fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
1335    let Some(object) = value.as_object() else {
1336        return Ok(None);
1337    };
1338    if !object.contains_key("codec") && !object.contains_key("blob") {
1339        return Ok(None);
1340    }
1341
1342    let codec = object
1343        .get("codec")
1344        .and_then(Value::as_str)
1345        .ok_or_else(invalid_payload_envelope)?;
1346    validate_payload_codec(codec)?;
1347    let blob = object
1348        .get("blob")
1349        .and_then(Value::as_str)
1350        .ok_or_else(invalid_payload_envelope)?;
1351    Ok(Some((codec, blob)))
1352}
1353
1354fn invalid_payload_envelope() -> Error {
1355    Error::Codec(
1356        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
1357            .to_string(),
1358    )
1359}
1360
1361fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
1362    for command in commands {
1363        let Some(command) = command.as_object() else {
1364            continue;
1365        };
1366        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
1367            continue;
1368        };
1369        let Some(payload_field) = workflow_command_payload_field(command_type) else {
1370            continue;
1371        };
1372
1373        if let Some(codec) = command.get("payload_codec") {
1374            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
1375            validate_payload_codec(codec)?;
1376        }
1377
1378        let payload = command
1379            .get(payload_field)
1380            .ok_or_else(invalid_payload_envelope)?;
1381        validate_outbound_payload_envelope(payload)?;
1382    }
1383    Ok(())
1384}
1385
1386fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
1387    match command_type {
1388        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
1389        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
1390        "start_service_operation" => Some("request_payload"),
1391        _ => None,
1392    }
1393}
1394
1395fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
1396    let Some((codec, blob)) = payload_envelope_parts(value)? else {
1397        return Err(untagged_payload_value());
1398    };
1399    validate_payload_codec(codec)?;
1400    decode_avro_value_blob(blob)?;
1401    Ok(())
1402}
1403
1404fn unsupported_payload_codec(codec: &str) -> Error {
1405    Error::Codec(format!(
1406        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1407    ))
1408}
1409
1410fn untagged_payload_value() -> Error {
1411    Error::Codec(
1412        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
1413            .to_string(),
1414    )
1415}
1416
1417fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1418    let bytes = BASE64.decode(blob).map_err(|err| {
1419        Error::Codec(format!(
1420            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1421        ))
1422    })?;
1423
1424    if serde_json::from_slice::<Value>(&bytes).is_ok() {
1425        return Err(unsupported_payload_codec("json"));
1426    }
1427
1428    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1429        return Err(Error::Codec(
1430            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1431        ));
1432    }
1433
1434    let fingerprint: [u8; 8] = bytes[2..10]
1435        .try_into()
1436        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1437    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1438        return Err(Error::Codec(format!(
1439            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1440            fingerprint
1441                .iter()
1442                .map(|byte| format!("{byte:02x}"))
1443                .collect::<String>()
1444        )));
1445    }
1446
1447    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1448    // The current fingerprint selects the current immutable schema, so reader
1449    // resolution would only re-walk the same recursive union. Future retained
1450    // writer fingerprints supply a distinct reader schema in this branch.
1451    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1452    if datum_reader.truncated {
1453        return Err(Error::Codec(
1454            "invalid_payload_framing: truncated Avro Value datum".to_string(),
1455        ));
1456    }
1457    let datum = datum.map_err(|err| {
1458        Error::Codec(format!(
1459            "invalid_payload_framing: malformed Avro Value datum: {err}"
1460        ))
1461    })?;
1462    if datum_reader.remaining() != 0 {
1463        return Err(Error::Codec(format!(
1464            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1465            datum_reader.remaining()
1466        )));
1467    }
1468    avro_value_from_datum(datum)
1469}
1470
1471struct StrictAvroDatumReader<'a> {
1472    bytes: &'a [u8],
1473    offset: usize,
1474    truncated: bool,
1475}
1476
1477impl<'a> StrictAvroDatumReader<'a> {
1478    fn new(bytes: &'a [u8]) -> Self {
1479        Self {
1480            bytes,
1481            offset: 0,
1482            truncated: false,
1483        }
1484    }
1485
1486    fn remaining(&self) -> usize {
1487        self.bytes.len() - self.offset
1488    }
1489}
1490
1491impl Read for StrictAvroDatumReader<'_> {
1492    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1493        let count = buffer.len().min(self.remaining());
1494        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1495        self.offset += count;
1496        if count < buffer.len() {
1497            self.truncated = true;
1498        }
1499
1500        Ok(count)
1501    }
1502}
1503
1504fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1505    let branch = match value {
1506        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1507        AvroValue::Boolean(value) => AvroDatum::Union(
1508            1,
1509            Box::new(AvroDatum::Record(vec![(
1510                "boolean".to_string(),
1511                AvroDatum::Boolean(*value),
1512            )])),
1513        ),
1514        AvroValue::Long(value) => AvroDatum::Union(
1515            2,
1516            Box::new(AvroDatum::Record(vec![(
1517                "long".to_string(),
1518                AvroDatum::Long(*value),
1519            )])),
1520        ),
1521        AvroValue::Double(value) => {
1522            if !value.is_finite() {
1523                return Err(Error::Codec(
1524                    "non_finite_float: Avro Value doubles must be finite".to_string(),
1525                ));
1526            }
1527            AvroDatum::Union(
1528                3,
1529                Box::new(AvroDatum::Record(vec![(
1530                    "double".to_string(),
1531                    AvroDatum::Double(*value),
1532                )])),
1533            )
1534        }
1535        AvroValue::Bytes(value) => AvroDatum::Union(
1536            4,
1537            Box::new(AvroDatum::Record(vec![(
1538                "bytes".to_string(),
1539                AvroDatum::Bytes(value.clone()),
1540            )])),
1541        ),
1542        AvroValue::String(value) => AvroDatum::Union(
1543            5,
1544            Box::new(AvroDatum::Record(vec![(
1545                "string".to_string(),
1546                AvroDatum::String(value.clone()),
1547            )])),
1548        ),
1549        AvroValue::Array(values) => AvroDatum::Union(
1550            6,
1551            Box::new(AvroDatum::Record(vec![(
1552                "items".to_string(),
1553                AvroDatum::Array(
1554                    values
1555                        .iter()
1556                        .map(avro_value_to_datum)
1557                        .collect::<Result<Vec<_>>>()?,
1558                ),
1559            )])),
1560        ),
1561        AvroValue::Map(values) => AvroDatum::Union(
1562            7,
1563            Box::new(AvroDatum::Record(vec![(
1564                "entries".to_string(),
1565                AvroDatum::Map(
1566                    values
1567                        .iter()
1568                        .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1569                        .collect::<Result<HashMap<_, _>>>()?,
1570                ),
1571            )])),
1572        ),
1573    };
1574    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1575}
1576
1577fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1578    let AvroDatum::Record(mut outer) = datum else {
1579        return Err(Error::Codec(
1580            "invalid_payload_framing: datum is not a Value record".to_string(),
1581        ));
1582    };
1583    let (_, branch) = outer
1584        .pop()
1585        .filter(|(name, _)| name == "value")
1586        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1587    let AvroDatum::Union(_, branch) = branch else {
1588        return Err(Error::Codec(
1589            "invalid_payload_framing: invalid Value union".to_string(),
1590        ));
1591    };
1592    match *branch {
1593        AvroDatum::Null => Ok(AvroValue::Null),
1594        AvroDatum::Record(mut fields) => {
1595            let (name, value) = fields.pop().ok_or_else(|| {
1596                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1597            })?;
1598            match (name.as_str(), value) {
1599                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1600                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1601                ("double", AvroDatum::Double(value)) if value.is_finite() => {
1602                    Ok(AvroValue::Double(value))
1603                }
1604                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1605                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1606                ("items", AvroDatum::Array(values)) => values
1607                    .into_iter()
1608                    .map(avro_value_from_datum)
1609                    .collect::<Result<Vec<_>>>()
1610                    .map(AvroValue::Array),
1611                ("entries", AvroDatum::Map(values)) => values
1612                    .into_iter()
1613                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1614                    .collect::<Result<BTreeMap<_, _>>>()
1615                    .map(AvroValue::Map),
1616                _ => Err(Error::Codec(
1617                    "invalid_payload_framing: unknown Value branch".to_string(),
1618                )),
1619            }
1620        }
1621        _ => Err(Error::Codec(
1622            "invalid_payload_framing: invalid Value branch".to_string(),
1623        )),
1624    }
1625}
1626
1627fn avro_value_schema() -> Result<&'static Schema> {
1628    match AVRO_VALUE_SCHEMA.get_or_init(|| {
1629        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1630            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1631    }) {
1632        Ok(schema) => Ok(schema),
1633        Err(message) => Err(Error::Codec(message.clone())),
1634    }
1635}
1636
1637#[derive(Clone, Debug)]
1638pub struct Client {
1639    http: reqwest::Client,
1640    base_url: String,
1641    token: Option<String>,
1642    control_token: Option<String>,
1643    worker_token: Option<String>,
1644    namespace: String,
1645}
1646
1647impl Client {
1648    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1649        Self::builder(base_url).build()
1650    }
1651
1652    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1653        ClientBuilder {
1654            base_url: base_url.into(),
1655            token: None,
1656            control_token: None,
1657            worker_token: None,
1658            namespace: "default".to_string(),
1659            timeout: Duration::from_secs(60),
1660        }
1661    }
1662
1663    pub async fn health(&self) -> Result<Value> {
1664        self.request_json(
1665            reqwest::Method::GET,
1666            "/health",
1667            RequestProtocol::ControlPlane,
1668            Option::<&Value>::None,
1669        )
1670        .await
1671    }
1672
1673    pub async fn cluster_info(&self) -> Result<Value> {
1674        self.request_json(
1675            reqwest::Method::GET,
1676            "/cluster/info",
1677            RequestProtocol::ControlPlane,
1678            Option::<&Value>::None,
1679        )
1680        .await
1681    }
1682
1683    pub async fn start_workflow<T: Serialize>(
1684        &self,
1685        workflow_type: &str,
1686        task_queue: &str,
1687        workflow_id: &str,
1688        input: T,
1689    ) -> Result<WorkflowHandle> {
1690        self.start_workflow_with_options(
1691            workflow_type,
1692            task_queue,
1693            workflow_id,
1694            WorkflowStartOptions::default(),
1695            input,
1696        )
1697        .await
1698    }
1699
1700    /// Start a workflow with explicit server-enforced execution and run
1701    /// deadlines.
1702    pub async fn start_workflow_with_options<T: Serialize>(
1703        &self,
1704        workflow_type: &str,
1705        task_queue: &str,
1706        workflow_id: &str,
1707        options: WorkflowStartOptions,
1708        input: T,
1709    ) -> Result<WorkflowHandle> {
1710        options.validate()?;
1711        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1712        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1713        let body = json!({
1714            "workflow_id": workflow_id,
1715            "workflow_type": workflow_type,
1716            "task_queue": task_queue,
1717            "input": input_envelope,
1718            "execution_timeout_seconds": options.execution_timeout_seconds,
1719            "run_timeout_seconds": options.run_timeout_seconds
1720        });
1721
1722        let data: Value = self
1723            .request_json(
1724                reqwest::Method::POST,
1725                "/workflows",
1726                RequestProtocol::ControlPlane,
1727                Some(&body),
1728            )
1729            .await?;
1730
1731        Ok(WorkflowHandle {
1732            client: self.clone(),
1733            workflow_id: data
1734                .get("workflow_id")
1735                .and_then(Value::as_str)
1736                .unwrap_or(workflow_id)
1737                .to_string(),
1738            run_id: data
1739                .get("run_id")
1740                .and_then(Value::as_str)
1741                .map(str::to_string),
1742            workflow_type: data
1743                .get("workflow_type")
1744                .and_then(Value::as_str)
1745                .unwrap_or(workflow_type)
1746                .to_string(),
1747        })
1748    }
1749
1750    pub async fn signal_workflow<T: Serialize>(
1751        &self,
1752        workflow_id: &str,
1753        signal_name: &str,
1754        input: T,
1755    ) -> Result<Value> {
1756        self.signal_workflow_target(workflow_id, None, signal_name, input)
1757            .await
1758    }
1759
1760    /// Signal only if `run_id` is still the current run for this instance.
1761    pub async fn signal_workflow_run<T: Serialize>(
1762        &self,
1763        workflow_id: &str,
1764        run_id: &str,
1765        signal_name: &str,
1766        input: T,
1767    ) -> Result<Value> {
1768        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1769            .await
1770    }
1771
1772    async fn signal_workflow_target<T: Serialize>(
1773        &self,
1774        workflow_id: &str,
1775        run_id: Option<&str>,
1776        signal_name: &str,
1777        input: T,
1778    ) -> Result<Value> {
1779        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1780        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1781        let body = json!({
1782            "input": input_envelope
1783        });
1784        let path = match run_id {
1785            Some(run_id) => {
1786                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1787            }
1788            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1789        };
1790        self.request_json(
1791            reqwest::Method::POST,
1792            &path,
1793            RequestProtocol::ControlPlane,
1794            Some(&body),
1795        )
1796        .await
1797    }
1798
1799    /// Request cooperative cancellation of the current run for an instance.
1800    pub async fn cancel_workflow(
1801        &self,
1802        workflow_id: &str,
1803        options: WorkflowCommandOptions,
1804    ) -> Result<WorkflowCommandResult> {
1805        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1806            .await
1807    }
1808
1809    /// Request cooperative cancellation only if `run_id` is still current.
1810    pub async fn cancel_workflow_run(
1811        &self,
1812        workflow_id: &str,
1813        run_id: &str,
1814        options: WorkflowCommandOptions,
1815    ) -> Result<WorkflowCommandResult> {
1816        self.workflow_command(
1817            workflow_id,
1818            Some(run_id),
1819            WorkflowCommandKind::Cancel,
1820            options,
1821        )
1822        .await
1823    }
1824
1825    /// Forcefully terminate the current run for an instance.
1826    pub async fn terminate_workflow(
1827        &self,
1828        workflow_id: &str,
1829        options: WorkflowCommandOptions,
1830    ) -> Result<WorkflowCommandResult> {
1831        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1832            .await
1833    }
1834
1835    /// Forcefully terminate only if `run_id` is still current.
1836    pub async fn terminate_workflow_run(
1837        &self,
1838        workflow_id: &str,
1839        run_id: &str,
1840        options: WorkflowCommandOptions,
1841    ) -> Result<WorkflowCommandResult> {
1842        self.workflow_command(
1843            workflow_id,
1844            Some(run_id),
1845            WorkflowCommandKind::Terminate,
1846            options,
1847        )
1848        .await
1849    }
1850
1851    async fn workflow_command(
1852        &self,
1853        workflow_id: &str,
1854        run_id: Option<&str>,
1855        command: WorkflowCommandKind,
1856        options: WorkflowCommandOptions,
1857    ) -> Result<WorkflowCommandResult> {
1858        let path = match run_id {
1859            Some(run_id) => format!(
1860                "/workflows/{workflow_id}/runs/{run_id}/{}",
1861                command.as_str()
1862            ),
1863            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1864        };
1865        let data = match self
1866            .request_json(
1867                reqwest::Method::POST,
1868                &path,
1869                RequestProtocol::ControlPlane,
1870                Some(&options),
1871            )
1872            .await
1873        {
1874            Ok(data) => data,
1875            Err(Error::Http { status, body }) => {
1876                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1877                    command,
1878                    status,
1879                    body,
1880                    workflow_id,
1881                    run_id,
1882                )));
1883            }
1884            Err(error) => return Err(error),
1885        };
1886
1887        Ok(workflow_command_result(command, data, workflow_id, run_id))
1888    }
1889
1890    /// Execute a named, read-only query against a running or completed workflow.
1891    ///
1892    /// Arguments and results use the platform payload envelope. Server and
1893    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1894    /// reason, HTTP status, and original response body.
1895    pub async fn query_workflow<T: Serialize>(
1896        &self,
1897        workflow_id: &str,
1898        query_name: &str,
1899        input: T,
1900    ) -> Result<Value> {
1901        self.query_workflow_target(workflow_id, None, query_name, input)
1902            .await
1903    }
1904
1905    /// Query only if `run_id` is still current, preventing accidental retargeting.
1906    pub async fn query_workflow_run<T: Serialize>(
1907        &self,
1908        workflow_id: &str,
1909        run_id: &str,
1910        query_name: &str,
1911        input: T,
1912    ) -> Result<Value> {
1913        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1914            .await
1915    }
1916
1917    /// Query a workflow and return the lossless fixed Avro Value result.
1918    pub async fn query_workflow_avro_value<T: Serialize>(
1919        &self,
1920        workflow_id: &str,
1921        query_name: &str,
1922        input: T,
1923    ) -> Result<AvroValue> {
1924        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1925            .await
1926    }
1927
1928    /// Query a selected run and return the lossless fixed Avro Value result.
1929    pub async fn query_workflow_run_avro_value<T: Serialize>(
1930        &self,
1931        workflow_id: &str,
1932        run_id: &str,
1933        query_name: &str,
1934        input: T,
1935    ) -> Result<AvroValue> {
1936        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1937            .await
1938    }
1939
1940    async fn query_workflow_avro_value_target<T: Serialize>(
1941        &self,
1942        workflow_id: &str,
1943        run_id: Option<&str>,
1944        query_name: &str,
1945        input: T,
1946    ) -> Result<AvroValue> {
1947        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1948        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1949        let path = match run_id {
1950            Some(run_id) => {
1951                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1952            }
1953            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1954        };
1955        let response: Value = match self
1956            .request_json(
1957                reqwest::Method::POST,
1958                &path,
1959                RequestProtocol::ControlPlane,
1960                Some(&body),
1961            )
1962            .await
1963        {
1964            Ok(response) => response,
1965            Err(Error::Http { status, body }) => {
1966                return Err(Error::QueryFailed(query_failure(status, body)));
1967            }
1968            Err(error) => return Err(error),
1969        };
1970
1971        let envelope = response
1972            .get("result_envelope")
1973            .filter(|envelope| !envelope.is_null())
1974            .ok_or_else(|| {
1975                Error::Codec(
1976                    "missing_payload_envelope: typed query result requires result_envelope"
1977                        .to_string(),
1978                )
1979            })?;
1980        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1981    }
1982
1983    async fn query_workflow_target<T: Serialize>(
1984        &self,
1985        workflow_id: &str,
1986        run_id: Option<&str>,
1987        query_name: &str,
1988        input: T,
1989    ) -> Result<Value> {
1990        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1991        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1992        let body = json!({
1993            "input": input_envelope
1994        });
1995        let path = match run_id {
1996            Some(run_id) => {
1997                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1998            }
1999            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2000        };
2001        let response: Value = match self
2002            .request_json(
2003                reqwest::Method::POST,
2004                &path,
2005                RequestProtocol::ControlPlane,
2006                Some(&body),
2007            )
2008            .await
2009        {
2010            Ok(response) => response,
2011            Err(Error::Http { status, body }) => {
2012                return Err(Error::QueryFailed(query_failure(status, body)));
2013            }
2014            Err(error) => return Err(error),
2015        };
2016
2017        if let Some(envelope) = response
2018            .get("result_envelope")
2019            .filter(|envelope| !envelope.is_null())
2020        {
2021            return decode_wire_value(envelope, DEFAULT_CODEC);
2022        }
2023
2024        Ok(response.get("result").cloned().unwrap_or(Value::Null))
2025    }
2026
2027    /// Send a synchronous update using fixed Avro Value arguments.
2028    pub async fn update_workflow<T: Serialize>(
2029        &self,
2030        workflow_id: &str,
2031        update_name: &str,
2032        input: T,
2033        request_id: Option<&str>,
2034    ) -> Result<Value> {
2035        let response = self
2036            .update_workflow_response(workflow_id, update_name, input, request_id)
2037            .await?;
2038        if let Some(envelope) = response
2039            .get("result_envelope")
2040            .filter(|envelope| !envelope.is_null())
2041        {
2042            return decode_wire_value(envelope, DEFAULT_CODEC);
2043        }
2044        Ok(response.get("result").cloned().unwrap_or(response))
2045    }
2046
2047    /// Send a synchronous update and retain a bytes-capable Avro result.
2048    pub async fn update_workflow_avro_value<T: Serialize>(
2049        &self,
2050        workflow_id: &str,
2051        update_name: &str,
2052        input: T,
2053        request_id: Option<&str>,
2054    ) -> Result<AvroValue> {
2055        let response = self
2056            .update_workflow_response(workflow_id, update_name, input, request_id)
2057            .await?;
2058        let envelope = response
2059            .get("result_envelope")
2060            .filter(|envelope| !envelope.is_null())
2061            .ok_or_else(|| {
2062                Error::Codec(
2063                    "missing_payload_envelope: typed update result requires result_envelope"
2064                        .to_string(),
2065                )
2066            })?;
2067        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2068    }
2069
2070    async fn update_workflow_response<T: Serialize>(
2071        &self,
2072        workflow_id: &str,
2073        update_name: &str,
2074        input: T,
2075        request_id: Option<&str>,
2076    ) -> Result<Value> {
2077        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2078        let mut body = json!({
2079            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
2080            "wait_for": "completed",
2081        });
2082        if let Some(request_id) = request_id {
2083            body["request_id"] = json!(request_id);
2084        }
2085        self.request_json(
2086            reqwest::Method::POST,
2087            &format!("/workflows/{workflow_id}/update/{update_name}"),
2088            RequestProtocol::ControlPlane,
2089            Some(&body),
2090        )
2091        .await
2092    }
2093
2094    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2095        let path = format!("/workflows/{workflow_id}");
2096        let mut data: WorkflowDescription = self
2097            .request_json(
2098                reqwest::Method::GET,
2099                &path,
2100                RequestProtocol::ControlPlane,
2101                Option::<&Value>::None,
2102            )
2103            .await?;
2104        data.decode_payloads()?;
2105        Ok(data)
2106    }
2107
2108    /// Describe one selected run, including historical terminal runs.
2109    pub async fn describe_workflow_run(
2110        &self,
2111        workflow_id: &str,
2112        run_id: &str,
2113    ) -> Result<WorkflowDescription> {
2114        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2115        let mut data: WorkflowDescription = self
2116            .request_json(
2117                reqwest::Method::GET,
2118                &path,
2119                RequestProtocol::ControlPlane,
2120                Option::<&Value>::None,
2121            )
2122            .await?;
2123        data.decode_payloads()?;
2124        Ok(data)
2125    }
2126
2127    pub async fn register_worker(
2128        &self,
2129        worker_id: &str,
2130        task_queue: &str,
2131        supported_workflow_types: Vec<String>,
2132        supported_activity_types: Vec<String>,
2133        max_concurrent_workflow_tasks: usize,
2134        max_concurrent_activity_tasks: usize,
2135    ) -> Result<RegisterWorkerResponse> {
2136        self.register_worker_with_capabilities(
2137            worker_id,
2138            task_queue,
2139            supported_workflow_types,
2140            supported_activity_types,
2141            max_concurrent_workflow_tasks,
2142            max_concurrent_activity_tasks,
2143            Vec::new(),
2144        )
2145        .await
2146    }
2147
2148    /// Register a worker and explicitly advertise additive worker capabilities.
2149    pub async fn register_worker_with_capabilities(
2150        &self,
2151        worker_id: &str,
2152        task_queue: &str,
2153        supported_workflow_types: Vec<String>,
2154        supported_activity_types: Vec<String>,
2155        max_concurrent_workflow_tasks: usize,
2156        max_concurrent_activity_tasks: usize,
2157        capabilities: Vec<String>,
2158    ) -> Result<RegisterWorkerResponse> {
2159        self.register_worker_with_command_contracts(
2160            worker_id,
2161            task_queue,
2162            supported_workflow_types,
2163            supported_activity_types,
2164            max_concurrent_workflow_tasks,
2165            max_concurrent_activity_tasks,
2166            capabilities,
2167            Value::Object(serde_json::Map::new()),
2168        )
2169        .await
2170    }
2171
2172    /// Register a worker and advertise its named query and update handlers.
2173    ///
2174    /// This Rust SDK cannot execute synchronous pre-accept update validation,
2175    /// so a workflow contract with a non-empty or malformed
2176    /// `update_validators` declaration returns
2177    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
2178    #[allow(clippy::too_many_arguments)]
2179    pub async fn register_worker_with_command_contracts(
2180        &self,
2181        worker_id: &str,
2182        task_queue: &str,
2183        supported_workflow_types: Vec<String>,
2184        supported_activity_types: Vec<String>,
2185        max_concurrent_workflow_tasks: usize,
2186        max_concurrent_activity_tasks: usize,
2187        capabilities: Vec<String>,
2188        workflow_command_contracts: Value,
2189    ) -> Result<RegisterWorkerResponse> {
2190        if let Some(contracts) = workflow_command_contracts.as_object() {
2191            for (workflow_type, contract) in contracts {
2192                let Some(update_validators) = contract.get("update_validators") else {
2193                    continue;
2194                };
2195                if !update_validators
2196                    .as_array()
2197                    .is_some_and(|validators| validators.is_empty())
2198                {
2199                    return Err(Error::UnsupportedUpdateValidators {
2200                        workflow_type: workflow_type.clone(),
2201                    });
2202                }
2203            }
2204        }
2205
2206        let mut body = json!({
2207            "worker_id": worker_id,
2208            "task_queue": task_queue,
2209            "runtime": "rust",
2210            "sdk_version": SDK_VERSION,
2211            "supported_workflow_types": supported_workflow_types,
2212            "supported_activity_types": supported_activity_types,
2213            "capabilities": capabilities,
2214            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2215            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2216        });
2217        if workflow_command_contracts
2218            .as_object()
2219            .is_some_and(|contracts| !contracts.is_empty())
2220        {
2221            body["workflow_command_contracts"] = workflow_command_contracts;
2222        }
2223
2224        self.request_json(
2225            reqwest::Method::POST,
2226            "/worker/register",
2227            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2228            Some(&body),
2229        )
2230        .await
2231    }
2232
2233    /// Gracefully remove one worker's registration through the worker plane.
2234    ///
2235    /// This operation is separate from operator-facing worker management. It
2236    /// uses worker-protocol authentication and returns the server's lease
2237    /// recovery result.
2238    pub async fn deregister_worker_registration(
2239        &self,
2240        worker_id: &str,
2241    ) -> Result<WorkerDeregistrationEnvelope> {
2242        let path = format!(
2243            "/worker/registrations/{}",
2244            percent_encode_path_segment(worker_id)
2245        );
2246        self.request_json(
2247            reqwest::Method::DELETE,
2248            &path,
2249            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2250            Option::<&Value>::None,
2251        )
2252        .await
2253    }
2254
2255    /// Long-poll for an ephemeral, read-only workflow query task.
2256    pub async fn poll_query_task(
2257        &self,
2258        worker_id: &str,
2259        task_queue: &str,
2260        timeout: Duration,
2261    ) -> Result<Option<QueryTask>> {
2262        Ok(self
2263            .poll_query_task_response(worker_id, task_queue, timeout)
2264            .await?
2265            .task)
2266    }
2267
2268    /// Poll a query task while preserving server stop and drain metadata.
2269    pub async fn poll_query_task_response(
2270        &self,
2271        worker_id: &str,
2272        task_queue: &str,
2273        timeout: Duration,
2274    ) -> Result<PollQueryTaskResponse> {
2275        let poll_request_id = unique_request_id("rust-query-poll");
2276        self.poll_query_task_response_with_request_id(
2277            worker_id,
2278            task_queue,
2279            timeout,
2280            &poll_request_id,
2281            1,
2282        )
2283        .await
2284    }
2285
2286    async fn poll_query_task_response_with_request_id(
2287        &self,
2288        worker_id: &str,
2289        task_queue: &str,
2290        timeout: Duration,
2291        poll_request_id: &str,
2292        transport_retries: usize,
2293    ) -> Result<PollQueryTaskResponse> {
2294        let timeout_seconds = long_poll_timeout_seconds(timeout);
2295        let body = json!({
2296            "worker_id": worker_id,
2297            "task_queue": task_queue,
2298            "poll_request_id": poll_request_id,
2299            "timeout_seconds": timeout_seconds,
2300        });
2301        self.poll_request_json(
2302            "/worker/query-tasks/poll",
2303            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2304            &body,
2305            timeout + Duration::from_secs(5),
2306            transport_retries,
2307        )
2308        .await
2309    }
2310
2311    /// Complete a query task without appending workflow history.
2312    pub async fn complete_query_task<T: Serialize>(
2313        &self,
2314        query_task_id: &str,
2315        lease_owner: &str,
2316        query_task_attempt: u64,
2317        result: T,
2318        codec: &str,
2319    ) -> Result<Value> {
2320        let typed_result = AvroValue::from_serialize(&result)?;
2321        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2322        self.complete_query_task_with_envelope(
2323            query_task_id,
2324            lease_owner,
2325            query_task_attempt,
2326            typed_result.into_json()?,
2327            result_envelope,
2328        )
2329        .await
2330    }
2331
2332    async fn complete_query_task_with_envelope(
2333        &self,
2334        query_task_id: &str,
2335        lease_owner: &str,
2336        query_task_attempt: u64,
2337        result: Value,
2338        result_envelope: Value,
2339    ) -> Result<Value> {
2340        let body = json!({
2341            "lease_owner": lease_owner,
2342            "query_task_attempt": query_task_attempt,
2343            "result": result,
2344            "result_envelope": result_envelope,
2345        });
2346        let path = format!("/worker/query-tasks/{query_task_id}/complete");
2347        let response = self
2348            .request_json(
2349                reqwest::Method::POST,
2350                &path,
2351                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2352                Some(&body),
2353            )
2354            .await;
2355        query_task_response(response)
2356    }
2357
2358    /// Report a stable machine-readable query-task failure.
2359    pub async fn fail_query_task(
2360        &self,
2361        query_task_id: &str,
2362        lease_owner: &str,
2363        query_task_attempt: u64,
2364        message: impl Into<String>,
2365        reason: impl Into<String>,
2366        failure_type: impl Into<String>,
2367    ) -> Result<Value> {
2368        let body = json!({
2369            "lease_owner": lease_owner,
2370            "query_task_attempt": query_task_attempt,
2371            "failure": {
2372                "message": message.into(),
2373                "reason": reason.into(),
2374                "type": failure_type.into(),
2375            }
2376        });
2377        let path = format!("/worker/query-tasks/{query_task_id}/fail");
2378        let response = self
2379            .request_json(
2380                reqwest::Method::POST,
2381                &path,
2382                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2383                Some(&body),
2384            )
2385            .await;
2386        query_task_response(response)
2387    }
2388
2389    pub async fn heartbeat_worker(
2390        &self,
2391        worker_id: &str,
2392        workflow_available: usize,
2393        activity_available: usize,
2394    ) -> Result<Value> {
2395        let body = json!({
2396            "worker_id": worker_id,
2397            "task_slots": {
2398                "workflow_available": workflow_available,
2399                "activity_available": activity_available
2400            },
2401            "process_metrics": {
2402                "process_id": std::process::id(),
2403                "process_uptime_seconds": 0
2404            }
2405        });
2406
2407        self.request_json(
2408            reqwest::Method::POST,
2409            "/worker/heartbeat",
2410            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2411            Some(&body),
2412        )
2413        .await
2414    }
2415
2416    pub async fn poll_workflow_task(
2417        &self,
2418        worker_id: &str,
2419        task_queue: &str,
2420        timeout: Duration,
2421    ) -> Result<Option<WorkflowTask>> {
2422        Ok(self
2423            .poll_workflow_task_response(worker_id, task_queue, timeout)
2424            .await?
2425            .task)
2426    }
2427
2428    pub async fn poll_workflow_task_response(
2429        &self,
2430        worker_id: &str,
2431        task_queue: &str,
2432        timeout: Duration,
2433    ) -> Result<PollWorkflowTaskResponse> {
2434        let poll_request_id = unique_request_id("rust-workflow-poll");
2435        self.poll_workflow_task_response_with_request_id(
2436            worker_id,
2437            task_queue,
2438            timeout,
2439            &poll_request_id,
2440            1,
2441        )
2442        .await
2443    }
2444
2445    async fn poll_workflow_task_response_with_request_id(
2446        &self,
2447        worker_id: &str,
2448        task_queue: &str,
2449        timeout: Duration,
2450        poll_request_id: &str,
2451        transport_retries: usize,
2452    ) -> Result<PollWorkflowTaskResponse> {
2453        let body = json!({
2454            "worker_id": worker_id,
2455            "task_queue": task_queue,
2456            "poll_request_id": poll_request_id,
2457            "timeout_seconds": long_poll_timeout_seconds(timeout),
2458        });
2459        let mut data: PollWorkflowTaskResponse = self
2460            .poll_request_json(
2461                "/worker/workflow-tasks/poll",
2462                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2463                &body,
2464                timeout + Duration::from_secs(5),
2465                transport_retries,
2466            )
2467            .await?;
2468
2469        if let Some(task) = data.task.as_mut() {
2470            self.fetch_remaining_workflow_history(worker_id, task)
2471                .await?;
2472        }
2473
2474        Ok(data)
2475    }
2476
2477    async fn fetch_remaining_workflow_history(
2478        &self,
2479        worker_id: &str,
2480        task: &mut WorkflowTask,
2481    ) -> Result<()> {
2482        let mut next_token = task.next_history_page_token.clone();
2483
2484        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2485            let lease_owner = task
2486                .lease_owner
2487                .clone()
2488                .unwrap_or_else(|| worker_id.to_string());
2489            let page = self
2490                .workflow_task_history_page(
2491                    &task.task_id,
2492                    &lease_owner,
2493                    task.workflow_task_attempt,
2494                    &token,
2495                )
2496                .await?;
2497
2498            task.append_history_page(page);
2499
2500            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2501                return Err(Error::Codec(
2502                    "workflow history pagination returned the same page token".to_string(),
2503                ));
2504            }
2505
2506            next_token = task.next_history_page_token.clone();
2507        }
2508
2509        Ok(())
2510    }
2511
2512    async fn workflow_task_history_page(
2513        &self,
2514        task_id: &str,
2515        lease_owner: &str,
2516        workflow_task_attempt: u64,
2517        next_history_page_token: &str,
2518    ) -> Result<WorkflowTaskHistoryPage> {
2519        let body = json!({
2520            "lease_owner": lease_owner,
2521            "workflow_task_attempt": workflow_task_attempt,
2522            "next_history_page_token": next_history_page_token
2523        });
2524        let path = format!("/worker/workflow-tasks/{task_id}/history");
2525
2526        self.request_json(
2527            reqwest::Method::POST,
2528            &path,
2529            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2530            Some(&body),
2531        )
2532        .await
2533    }
2534
2535    pub async fn complete_workflow_task(
2536        &self,
2537        task_id: &str,
2538        lease_owner: &str,
2539        workflow_task_attempt: u64,
2540        commands: Vec<Value>,
2541    ) -> Result<Value> {
2542        validate_workflow_task_commands(&commands)?;
2543        let body = json!({
2544            "lease_owner": lease_owner,
2545            "workflow_task_attempt": workflow_task_attempt,
2546            "commands": commands
2547        });
2548        let path = format!("/worker/workflow-tasks/{task_id}/complete");
2549        self.request_json(
2550            reqwest::Method::POST,
2551            &path,
2552            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2553            Some(&body),
2554        )
2555        .await
2556    }
2557
2558    pub async fn fail_workflow_task(
2559        &self,
2560        task_id: &str,
2561        lease_owner: &str,
2562        workflow_task_attempt: u64,
2563        message: impl Into<String>,
2564    ) -> Result<Value> {
2565        self.fail_workflow_task_with_type(
2566            task_id,
2567            lease_owner,
2568            workflow_task_attempt,
2569            message,
2570            "RustWorkflowTaskFailure",
2571        )
2572        .await
2573    }
2574
2575    async fn fail_workflow_task_with_type(
2576        &self,
2577        task_id: &str,
2578        lease_owner: &str,
2579        workflow_task_attempt: u64,
2580        message: impl Into<String>,
2581        failure_type: &str,
2582    ) -> Result<Value> {
2583        let body = json!({
2584            "lease_owner": lease_owner,
2585            "workflow_task_attempt": workflow_task_attempt,
2586            "failure": {
2587                "message": message.into(),
2588                "type": failure_type
2589            }
2590        });
2591        let path = format!("/worker/workflow-tasks/{task_id}/fail");
2592        self.request_json(
2593            reqwest::Method::POST,
2594            &path,
2595            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2596            Some(&body),
2597        )
2598        .await
2599    }
2600
2601    pub async fn poll_activity_task(
2602        &self,
2603        worker_id: &str,
2604        task_queue: &str,
2605        timeout: Duration,
2606    ) -> Result<Option<ActivityTask>> {
2607        Ok(self
2608            .poll_activity_task_response(worker_id, task_queue, timeout)
2609            .await?
2610            .task)
2611    }
2612
2613    /// Poll an activity task while preserving server stop and drain metadata.
2614    pub async fn poll_activity_task_response(
2615        &self,
2616        worker_id: &str,
2617        task_queue: &str,
2618        timeout: Duration,
2619    ) -> Result<PollActivityTaskResponse> {
2620        let poll_request_id = unique_request_id("rust-activity-poll");
2621        self.poll_activity_task_response_with_request_id(
2622            worker_id,
2623            task_queue,
2624            timeout,
2625            &poll_request_id,
2626            1,
2627        )
2628        .await
2629    }
2630
2631    async fn poll_activity_task_response_with_request_id(
2632        &self,
2633        worker_id: &str,
2634        task_queue: &str,
2635        timeout: Duration,
2636        poll_request_id: &str,
2637        transport_retries: usize,
2638    ) -> Result<PollActivityTaskResponse> {
2639        let body = json!({
2640            "worker_id": worker_id,
2641            "task_queue": task_queue,
2642            "poll_request_id": poll_request_id,
2643            "timeout_seconds": long_poll_timeout_seconds(timeout),
2644        });
2645        let data: PollActivityTaskResponse = self
2646            .poll_request_json(
2647                "/worker/activity-tasks/poll",
2648                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2649                &body,
2650                timeout + Duration::from_secs(5),
2651                transport_retries,
2652            )
2653            .await?;
2654        Ok(data)
2655    }
2656
2657    pub async fn complete_activity_task<T: Serialize>(
2658        &self,
2659        task_id: &str,
2660        activity_attempt_id: &str,
2661        lease_owner: &str,
2662        result: T,
2663        codec: &str,
2664    ) -> Result<Value> {
2665        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2666        let body = json!({
2667            "activity_attempt_id": activity_attempt_id,
2668            "lease_owner": lease_owner,
2669            "result": result
2670        });
2671        let path = format!("/worker/activity-tasks/{task_id}/complete");
2672        activity_task_response(
2673            self.request_json(
2674                reqwest::Method::POST,
2675                &path,
2676                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2677                Some(&body),
2678            )
2679            .await,
2680            "complete",
2681            task_id,
2682            activity_attempt_id,
2683        )
2684    }
2685
2686    pub async fn fail_activity_task(
2687        &self,
2688        task_id: &str,
2689        activity_attempt_id: &str,
2690        lease_owner: &str,
2691        message: impl Into<String>,
2692        non_retryable: bool,
2693    ) -> Result<Value> {
2694        let body = json!({
2695            "activity_attempt_id": activity_attempt_id,
2696            "lease_owner": lease_owner,
2697            "failure": {
2698                "message": message.into(),
2699                "type": "RustActivityFailure",
2700                "non_retryable": non_retryable
2701            }
2702        });
2703        let path = format!("/worker/activity-tasks/{task_id}/fail");
2704        activity_task_response(
2705            self.request_json(
2706                reqwest::Method::POST,
2707                &path,
2708                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2709                Some(&body),
2710            )
2711            .await,
2712            "fail",
2713            task_id,
2714            activity_attempt_id,
2715        )
2716    }
2717
2718    pub async fn heartbeat_activity_task<T: Serialize>(
2719        &self,
2720        task_id: &str,
2721        activity_attempt_id: &str,
2722        lease_owner: &str,
2723        details: T,
2724    ) -> Result<ActivityHeartbeatResponse> {
2725        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2726        let body = json!({
2727            "activity_attempt_id": activity_attempt_id,
2728            "lease_owner": lease_owner,
2729            "details": details
2730        });
2731        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2732        activity_task_response(
2733            self.request_json(
2734                reqwest::Method::POST,
2735                &path,
2736                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2737                Some(&body),
2738            )
2739            .await,
2740            "heartbeat",
2741            task_id,
2742            activity_attempt_id,
2743        )
2744    }
2745
2746    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2747        &self,
2748        method: reqwest::Method,
2749        path: &str,
2750        protocol: RequestProtocol,
2751        body: Option<&B>,
2752    ) -> Result<T> {
2753        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2754            .await
2755    }
2756
2757    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2758        &self,
2759        method: reqwest::Method,
2760        path: &str,
2761        protocol: RequestProtocol,
2762        body: Option<&B>,
2763        timeout: Duration,
2764    ) -> Result<T> {
2765        let auth_token = self.auth_token(protocol)?;
2766        let mut request = self
2767            .http
2768            .request(method, format!("{}/api{}", self.base_url, path))
2769            .timeout(timeout)
2770            .header(reqwest::header::ACCEPT, "application/json")
2771            .header(reqwest::header::CONTENT_TYPE, "application/json")
2772            .header("X-Namespace", &self.namespace);
2773
2774        match protocol {
2775            RequestProtocol::Worker(version) => {
2776                request = request.header("X-Durable-Workflow-Protocol-Version", version);
2777            }
2778            RequestProtocol::ControlPlane => {
2779                request = request.header(
2780                    "X-Durable-Workflow-Control-Plane-Version",
2781                    CONTROL_PLANE_VERSION,
2782                );
2783            }
2784        }
2785
2786        if let Some(token) = auth_token {
2787            request = request.bearer_auth(token);
2788        }
2789
2790        if let Some(body) = body {
2791            request = request.json(body);
2792        }
2793
2794        let response = request.send().await?;
2795        let status = response.status();
2796        let bytes = response.bytes().await?;
2797
2798        if !status.is_success() {
2799            let body = String::from_utf8_lossy(&bytes).to_string();
2800            if let Some(protocol) = protocol_failure(status, &body) {
2801                return Err(Error::Protocol(protocol));
2802            }
2803            return Err(Error::Http { status, body });
2804        }
2805
2806        if bytes.is_empty() {
2807            return Ok(serde_json::from_value(Value::Null)?);
2808        }
2809
2810        Ok(serde_json::from_slice(&bytes)?)
2811    }
2812
2813    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2814        &self,
2815        path: &str,
2816        protocol: RequestProtocol,
2817        body: &B,
2818        timeout: Duration,
2819        max_retries: usize,
2820    ) -> Result<T> {
2821        let mut retries = 0;
2822
2823        loop {
2824            let response = self
2825                .request_json_with_timeout(
2826                    reqwest::Method::POST,
2827                    path,
2828                    protocol,
2829                    Some(body),
2830                    timeout,
2831                )
2832                .await;
2833
2834            match response {
2835                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2836                response => return worker_poll_response(response),
2837            }
2838        }
2839    }
2840
2841    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2842        match protocol {
2843            RequestProtocol::Worker(_) => {
2844                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2845                    return Ok(Some(token));
2846                }
2847                if self.control_token.is_some() {
2848                    return Err(Error::MissingRoleCredentials {
2849                        role: "worker",
2850                        opposite_role: "control",
2851                    });
2852                }
2853                Ok(None)
2854            }
2855            RequestProtocol::ControlPlane => {
2856                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2857                    return Ok(Some(token));
2858                }
2859                if self.worker_token.is_some() {
2860                    return Err(Error::MissingRoleCredentials {
2861                        role: "control",
2862                        opposite_role: "worker",
2863                    });
2864                }
2865                Ok(None)
2866            }
2867        }
2868    }
2869}
2870
2871fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2872    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2873    let reason = body
2874        .get("reason")
2875        .and_then(Value::as_str)
2876        .unwrap_or("query_rejected")
2877        .to_string();
2878    let message = body
2879        .get("message")
2880        .or_else(|| body.get("error"))
2881        .and_then(Value::as_str)
2882        .unwrap_or("workflow query was rejected")
2883        .to_string();
2884
2885    QueryFailure {
2886        status: status.as_u16(),
2887        reason,
2888        message,
2889        body,
2890    }
2891}
2892
2893fn workflow_command_result(
2894    command: WorkflowCommandKind,
2895    data: Value,
2896    workflow_id: &str,
2897    run_id: Option<&str>,
2898) -> WorkflowCommandResult {
2899    WorkflowCommandResult {
2900        command,
2901        workflow_id: data
2902            .get("workflow_id")
2903            .and_then(Value::as_str)
2904            .unwrap_or(workflow_id)
2905            .to_string(),
2906        run_id: data
2907            .get("run_id")
2908            .and_then(Value::as_str)
2909            .or(run_id)
2910            .map(str::to_string),
2911        outcome: data
2912            .get("outcome")
2913            .and_then(Value::as_str)
2914            .map(str::to_string),
2915        reason: data
2916            .get("reason")
2917            .and_then(Value::as_str)
2918            .map(str::to_string),
2919        command_status: data
2920            .get("command_status")
2921            .and_then(Value::as_str)
2922            .map(str::to_string),
2923        raw: data,
2924    }
2925}
2926
2927fn workflow_command_rejection(
2928    command: WorkflowCommandKind,
2929    status: reqwest::StatusCode,
2930    raw_body: String,
2931    workflow_id: &str,
2932    run_id: Option<&str>,
2933) -> WorkflowCommandRejection {
2934    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2935    WorkflowCommandRejection {
2936        command,
2937        status: status.as_u16(),
2938        reason: body
2939            .get("reason")
2940            .and_then(Value::as_str)
2941            .unwrap_or("workflow_command_rejected")
2942            .to_string(),
2943        message: body
2944            .get("message")
2945            .or_else(|| body.get("error"))
2946            .and_then(Value::as_str)
2947            .unwrap_or("workflow lifecycle command was rejected")
2948            .to_string(),
2949        workflow_id: body
2950            .get("workflow_id")
2951            .and_then(Value::as_str)
2952            .unwrap_or(workflow_id)
2953            .to_string(),
2954        run_id: body
2955            .get("run_id")
2956            .and_then(Value::as_str)
2957            .or(run_id)
2958            .map(str::to_string),
2959        target_scope: body
2960            .get("target_scope")
2961            .and_then(Value::as_str)
2962            .map(str::to_string),
2963        body,
2964    }
2965}
2966
2967fn query_task_response(response: Result<Value>) -> Result<Value> {
2968    match response {
2969        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2970        response => response,
2971    }
2972}
2973
2974fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2975    match response {
2976        Err(Error::Http { status, body })
2977            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2978        {
2979            Ok(serde_json::from_str(&body)?)
2980        }
2981        response => response,
2982    }
2983}
2984
2985fn worker_poll_body_is_stop(body: &str) -> bool {
2986    serde_json::from_str::<Value>(body)
2987        .ok()
2988        .is_some_and(|body| {
2989            worker_poll_is_stop(
2990                body.get("poll_status").and_then(Value::as_str),
2991                body.get("reason").and_then(Value::as_str),
2992            )
2993        })
2994}
2995
2996fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2997    matches!(poll_status, Some("draining" | "stopped"))
2998        || matches!(reason, Some("worker_draining" | "worker_stopped"))
2999}
3000
3001fn query_task_rejection_is_final(error: &Error) -> bool {
3002    matches!(
3003        error,
3004        Error::QueryFailed(failure)
3005            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
3006    )
3007}
3008
3009fn activity_task_response<T>(
3010    response: Result<T>,
3011    operation: &str,
3012    task_id: &str,
3013    activity_attempt_id: &str,
3014) -> Result<T> {
3015    match response {
3016        Err(Error::Http { status, body }) => {
3017            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
3018            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
3019                operation: operation.to_string(),
3020                status: status.as_u16(),
3021                reason: body
3022                    .get("reason")
3023                    .and_then(Value::as_str)
3024                    .unwrap_or("activity_task_rejected")
3025                    .to_string(),
3026                task_id: body
3027                    .get("task_id")
3028                    .and_then(Value::as_str)
3029                    .unwrap_or(task_id)
3030                    .to_string(),
3031                activity_attempt_id: body
3032                    .get("activity_attempt_id")
3033                    .and_then(Value::as_str)
3034                    .unwrap_or(activity_attempt_id)
3035                    .to_string(),
3036                cancel_requested: body
3037                    .get("cancel_requested")
3038                    .and_then(Value::as_bool)
3039                    .unwrap_or(false),
3040                can_continue: body.get("can_continue").and_then(Value::as_bool),
3041                run_closed_reason: body
3042                    .get("run_closed_reason")
3043                    .and_then(Value::as_str)
3044                    .map(str::to_string),
3045                body,
3046            }))
3047        }
3048        response => response,
3049    }
3050}
3051
3052fn activity_task_rejection_is_final(error: &Error) -> bool {
3053    matches!(
3054        error,
3055        Error::ActivityTaskRejected(rejection)
3056            if matches!(
3057                rejection.reason.as_str(),
3058                "run_cancelled"
3059                    | "run_terminated"
3060                    | "attempt_closed"
3061                    | "stale_attempt"
3062                    | "activity_cancelled"
3063                    | "task_cancelled"
3064                    | "run_closed"
3065                    | "activity_not_running"
3066                    | "attempt_not_found"
3067            )
3068    )
3069}
3070
3071fn workflow_task_completion_is_terminal_timeout(
3072    error: &Error,
3073    task_id: &str,
3074    workflow_task_attempt: u64,
3075    run_id: Option<&str>,
3076) -> bool {
3077    let Error::Http { status, body } = error else {
3078        return false;
3079    };
3080    if *status != reqwest::StatusCode::CONFLICT {
3081        return false;
3082    }
3083
3084    let Some(run_id) = run_id else {
3085        return false;
3086    };
3087    let Ok(body) = serde_json::from_str::<Value>(body) else {
3088        return false;
3089    };
3090
3091    body.get("recorded").and_then(Value::as_bool) == Some(false)
3092        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
3093        && body.get("run_status").and_then(Value::as_str) == Some("failed")
3094        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
3095        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
3096        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
3097}
3098
3099fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
3100    let body: Value = serde_json::from_str(raw_body).ok()?;
3101    let reason = body.get("reason")?.as_str()?;
3102    if !matches!(
3103        reason,
3104        "missing_protocol_version"
3105            | "unsupported_protocol_version"
3106            | "missing_control_plane_version"
3107            | "unsupported_control_plane_version"
3108    ) {
3109        return None;
3110    }
3111
3112    Some(ProtocolFailure {
3113        status: status.as_u16(),
3114        reason: reason.to_string(),
3115        message: body
3116            .get("message")
3117            .or_else(|| body.get("error"))
3118            .and_then(Value::as_str)
3119            .unwrap_or("protocol version rejected")
3120            .to_string(),
3121        supported_version: body
3122            .get("supported_version")
3123            .and_then(Value::as_str)
3124            .map(str::to_string),
3125        requested_version: body
3126            .get("requested_version")
3127            .and_then(Value::as_str)
3128            .map(str::to_string),
3129        body,
3130    })
3131}
3132
3133fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3134    timeout
3135        .as_secs()
3136        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3137        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3138}
3139
3140fn worker_operation_is_retryable(error: &Error) -> bool {
3141    match error {
3142        Error::Transport(error) => {
3143            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3144        }
3145        Error::Http { status, .. } => {
3146            matches!(
3147                *status,
3148                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3149            ) || status.is_server_error()
3150        }
3151        _ => false,
3152    }
3153}
3154
3155fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3156    let exponent = retry.saturating_sub(1).min(31) as u32;
3157    policy
3158        .initial_backoff
3159        .saturating_mul(1_u32 << exponent)
3160        .min(policy.max_backoff)
3161}
3162
3163#[derive(Debug)]
3164pub struct ClientBuilder {
3165    base_url: String,
3166    token: Option<String>,
3167    control_token: Option<String>,
3168    worker_token: Option<String>,
3169    namespace: String,
3170    timeout: Duration,
3171}
3172
3173impl ClientBuilder {
3174    pub fn token(mut self, token: Option<String>) -> Self {
3175        self.token = token;
3176        self
3177    }
3178
3179    pub fn control_token(mut self, token: Option<String>) -> Self {
3180        self.control_token = token;
3181        self
3182    }
3183
3184    pub fn worker_token(mut self, token: Option<String>) -> Self {
3185        self.worker_token = token;
3186        self
3187    }
3188
3189    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3190        self.namespace = namespace.into();
3191        self
3192    }
3193
3194    pub fn timeout(mut self, timeout: Duration) -> Self {
3195        self.timeout = timeout;
3196        self
3197    }
3198
3199    pub fn build(self) -> Result<Client> {
3200        let base_url = self.base_url.trim_end_matches('/').to_string();
3201        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3202            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3203            .unwrap_or_else(|_| base_url.ends_with("/api"));
3204
3205        if has_sdk_api_suffix {
3206            return Err(Error::InvalidBaseUrl);
3207        }
3208
3209        Ok(Client {
3210            http: reqwest::Client::builder().timeout(self.timeout).build()?,
3211            base_url,
3212            token: self.token,
3213            control_token: self.control_token,
3214            worker_token: self.worker_token,
3215            namespace: self.namespace,
3216        })
3217    }
3218}
3219
3220#[derive(Clone, Debug)]
3221pub struct WorkflowHandle {
3222    client: Client,
3223    pub workflow_id: String,
3224    pub run_id: Option<String>,
3225    pub workflow_type: String,
3226}
3227
3228impl WorkflowHandle {
3229    /// Describe whichever run is current for this stable workflow instance.
3230    pub async fn describe(&self) -> Result<WorkflowDescription> {
3231        self.client.describe_workflow(&self.workflow_id).await
3232    }
3233
3234    /// Describe the run identity originally selected by this handle.
3235    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3236        let run_id = self.run_id.as_deref().ok_or_else(|| {
3237            Error::Codec("run_id is required for selected-run description".to_string())
3238        })?;
3239        self.client
3240            .describe_workflow_run(&self.workflow_id, run_id)
3241            .await
3242    }
3243
3244    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3245        self.client
3246            .signal_workflow(&self.workflow_id, signal_name, input)
3247            .await
3248    }
3249
3250    /// Signal only if this handle's selected run is still current.
3251    pub async fn signal_selected_run<T: Serialize>(
3252        &self,
3253        signal_name: &str,
3254        input: T,
3255    ) -> Result<Value> {
3256        let run_id = self.run_id.as_deref().ok_or_else(|| {
3257            Error::Codec("run_id is required for selected-run signaling".to_string())
3258        })?;
3259        self.client
3260            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3261            .await
3262    }
3263
3264    /// Request cooperative cancellation of whichever run is current.
3265    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3266        self.client
3267            .cancel_workflow(&self.workflow_id, options)
3268            .await
3269    }
3270
3271    /// Request cancellation only if this handle's selected run is still current.
3272    pub async fn cancel_selected_run(
3273        &self,
3274        options: WorkflowCommandOptions,
3275    ) -> Result<WorkflowCommandResult> {
3276        let run_id = self.run_id.as_deref().ok_or_else(|| {
3277            Error::Codec("run_id is required for selected-run cancellation".to_string())
3278        })?;
3279        self.client
3280            .cancel_workflow_run(&self.workflow_id, run_id, options)
3281            .await
3282    }
3283
3284    /// Forcefully terminate whichever run is current.
3285    pub async fn terminate(
3286        &self,
3287        options: WorkflowCommandOptions,
3288    ) -> Result<WorkflowCommandResult> {
3289        self.client
3290            .terminate_workflow(&self.workflow_id, options)
3291            .await
3292    }
3293
3294    /// Terminate only if this handle's selected run is still current.
3295    pub async fn terminate_selected_run(
3296        &self,
3297        options: WorkflowCommandOptions,
3298    ) -> Result<WorkflowCommandResult> {
3299        let run_id = self.run_id.as_deref().ok_or_else(|| {
3300            Error::Codec("run_id is required for selected-run termination".to_string())
3301        })?;
3302        self.client
3303            .terminate_workflow_run(&self.workflow_id, run_id, options)
3304            .await
3305    }
3306
3307    /// Execute a named, read-only query against this workflow.
3308    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3309        self.client
3310            .query_workflow(&self.workflow_id, query_name, input)
3311            .await
3312    }
3313
3314    pub async fn query_avro_value<T: Serialize>(
3315        &self,
3316        query_name: &str,
3317        input: T,
3318    ) -> Result<AvroValue> {
3319        self.client
3320            .query_workflow_avro_value(&self.workflow_id, query_name, input)
3321            .await
3322    }
3323
3324    pub async fn update<T: Serialize>(
3325        &self,
3326        update_name: &str,
3327        input: T,
3328        request_id: Option<&str>,
3329    ) -> Result<Value> {
3330        self.client
3331            .update_workflow(&self.workflow_id, update_name, input, request_id)
3332            .await
3333    }
3334
3335    pub async fn update_avro_value<T: Serialize>(
3336        &self,
3337        update_name: &str,
3338        input: T,
3339        request_id: Option<&str>,
3340    ) -> Result<AvroValue> {
3341        self.client
3342            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3343            .await
3344    }
3345
3346    /// Query only if this handle's selected run is still current.
3347    pub async fn query_selected_run<T: Serialize>(
3348        &self,
3349        query_name: &str,
3350        input: T,
3351    ) -> Result<Value> {
3352        let run_id = self
3353            .run_id
3354            .as_deref()
3355            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3356        self.client
3357            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3358            .await
3359    }
3360
3361    /// Await the final terminal outcome of the current continue-as-new chain.
3362    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3363        self.result_target(options, None).await
3364    }
3365
3366    /// Await the final result without projecting Avro bytes through JSON.
3367    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3368        self.result_avro_value_target(options, None).await
3369    }
3370
3371    /// Await only the run identity originally selected by this handle.
3372    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3373        let run_id = self.run_id.as_deref().ok_or_else(|| {
3374            Error::Codec("run_id is required for selected-run result".to_string())
3375        })?;
3376        self.result_target(options, Some(run_id)).await
3377    }
3378
3379    /// Await the selected run's result on the lossless Avro Value surface.
3380    pub async fn result_selected_run_avro_value(
3381        &self,
3382        options: WorkflowResultOptions,
3383    ) -> Result<AvroValue> {
3384        let run_id = self.run_id.as_deref().ok_or_else(|| {
3385            Error::Codec("run_id is required for selected-run result".to_string())
3386        })?;
3387        self.result_avro_value_target(options, Some(run_id)).await
3388    }
3389
3390    async fn result_avro_value_target(
3391        &self,
3392        options: WorkflowResultOptions,
3393        selected_run_id: Option<&str>,
3394    ) -> Result<AvroValue> {
3395        let started = Instant::now();
3396
3397        loop {
3398            let description = match selected_run_id {
3399                Some(run_id) => {
3400                    self.client
3401                        .describe_workflow_run(&self.workflow_id, run_id)
3402                        .await?
3403                }
3404                None => self.describe().await?,
3405            };
3406            if description.is_completed() {
3407                return description.output_avro_value.ok_or_else(|| {
3408                    Error::Codec(
3409                        "missing_payload_envelope: typed workflow result requires output_envelope"
3410                            .to_string(),
3411                    )
3412                });
3413            }
3414            if description.is_terminal() {
3415                let outcome =
3416                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3417                return Err(match outcome.kind {
3418                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3419                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3420                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3421                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3422                });
3423            }
3424            if started.elapsed() >= options.timeout {
3425                return Err(Error::Timeout);
3426            }
3427            tokio::time::sleep(options.poll_interval).await;
3428        }
3429    }
3430
3431    async fn result_target(
3432        &self,
3433        options: WorkflowResultOptions,
3434        selected_run_id: Option<&str>,
3435    ) -> Result<Value> {
3436        let started = Instant::now();
3437
3438        loop {
3439            let description = match selected_run_id {
3440                Some(run_id) => {
3441                    self.client
3442                        .describe_workflow_run(&self.workflow_id, run_id)
3443                        .await?
3444                }
3445                None => self.describe().await?,
3446            };
3447            if description.is_completed() {
3448                return Ok(description.output.unwrap_or(Value::Null));
3449            }
3450
3451            if description.is_terminal() {
3452                let outcome =
3453                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3454                return Err(match outcome.kind {
3455                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3456                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3457                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3458                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3459                });
3460            }
3461
3462            if started.elapsed() >= options.timeout {
3463                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3464                    kind: WorkflowTerminalKind::TimedOut,
3465                    workflow_id: description
3466                        .workflow_id
3467                        .clone()
3468                        .unwrap_or_else(|| self.workflow_id.clone()),
3469                    run_id: description
3470                        .run_id
3471                        .clone()
3472                        .or_else(|| selected_run_id.map(str::to_string)),
3473                    reason: "result_wait_timeout".to_string(),
3474                    failure_category: Some("client_timeout".to_string()),
3475                    failure_id: None,
3476                    exception_type: None,
3477                    exception_class: None,
3478                    non_retryable: None,
3479                    message: Some(format!(
3480                        "workflow result was not terminal within {:?}",
3481                        options.timeout
3482                    )),
3483                    exception: None,
3484                    raw: description.raw_value(),
3485                }));
3486            }
3487
3488            tokio::time::sleep(options.poll_interval).await;
3489        }
3490    }
3491}
3492
3493#[derive(Clone, Copy, Debug)]
3494pub struct WorkflowResultOptions {
3495    pub poll_interval: Duration,
3496    pub timeout: Duration,
3497}
3498
3499impl Default for WorkflowResultOptions {
3500    fn default() -> Self {
3501        Self {
3502            poll_interval: Duration::from_millis(500),
3503            timeout: Duration::from_secs(30),
3504        }
3505    }
3506}
3507
3508#[derive(Clone, Debug, Deserialize)]
3509pub struct WorkflowDescription {
3510    pub workflow_id: Option<String>,
3511    pub run_id: Option<String>,
3512    pub workflow_type: Option<String>,
3513    pub status: Option<String>,
3514    #[serde(default)]
3515    pub closed_reason: Option<String>,
3516    #[serde(default)]
3517    pub error: Option<String>,
3518    #[serde(default)]
3519    pub failure: Option<Value>,
3520    #[serde(default)]
3521    pub exception: Option<Value>,
3522    #[serde(default)]
3523    pub failures: Vec<Value>,
3524    #[serde(default)]
3525    pub output: Option<Value>,
3526    #[serde(default)]
3527    pub output_envelope: Option<Value>,
3528    #[serde(skip)]
3529    pub output_avro_value: Option<AvroValue>,
3530    #[serde(flatten)]
3531    pub raw: HashMap<String, Value>,
3532}
3533
3534impl WorkflowDescription {
3535    pub fn is_completed(&self) -> bool {
3536        matches!(self.status.as_deref(), Some("completed" | "Completed"))
3537    }
3538
3539    pub fn is_terminal(&self) -> bool {
3540        matches!(
3541            self.status.as_deref(),
3542            Some(
3543                "completed"
3544                    | "Completed"
3545                    | "failed"
3546                    | "Failed"
3547                    | "cancelled"
3548                    | "Cancelled"
3549                    | "terminated"
3550                    | "Terminated"
3551                    | "timed_out"
3552                    | "TimedOut",
3553            )
3554        )
3555    }
3556
3557    fn decode_payloads(&mut self) -> Result<()> {
3558        if let Some(envelope) = &self.output_envelope {
3559            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3560            self.output = Some(value.clone().into_json()?);
3561            self.output_avro_value = Some(value);
3562        }
3563
3564        Ok(())
3565    }
3566
3567    fn raw_value(&self) -> Value {
3568        let mut data = self.raw.clone();
3569        data.insert(
3570            "workflow_id".to_string(),
3571            self.workflow_id
3572                .clone()
3573                .map(Value::String)
3574                .unwrap_or(Value::Null),
3575        );
3576        data.insert(
3577            "run_id".to_string(),
3578            self.run_id
3579                .clone()
3580                .map(Value::String)
3581                .unwrap_or(Value::Null),
3582        );
3583        data.insert(
3584            "workflow_type".to_string(),
3585            self.workflow_type
3586                .clone()
3587                .map(Value::String)
3588                .unwrap_or(Value::Null),
3589        );
3590        data.insert(
3591            "status".to_string(),
3592            self.status
3593                .clone()
3594                .map(Value::String)
3595                .unwrap_or(Value::Null),
3596        );
3597        data.insert(
3598            "closed_reason".to_string(),
3599            self.closed_reason
3600                .clone()
3601                .map(Value::String)
3602                .unwrap_or(Value::Null),
3603        );
3604        if let Some(failure) = &self.failure {
3605            data.insert("failure".to_string(), failure.clone());
3606        }
3607        if let Some(exception) = &self.exception {
3608            data.insert("exception".to_string(), exception.clone());
3609        }
3610        Value::Object(data.into_iter().collect())
3611    }
3612}
3613
3614fn workflow_terminal_outcome(
3615    description: &WorkflowDescription,
3616    workflow_id: &str,
3617    run_id: Option<&str>,
3618) -> WorkflowTerminalOutcome {
3619    let terminal_kind = description
3620        .closed_reason
3621        .as_deref()
3622        .or(description.status.as_deref())
3623        .unwrap_or("failed")
3624        .to_ascii_lowercase();
3625    let kind = match terminal_kind.as_str() {
3626        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3627        "terminated" => WorkflowTerminalKind::Terminated,
3628        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3629        _ => WorkflowTerminalKind::Failed,
3630    };
3631    let default_reason = match kind {
3632        WorkflowTerminalKind::Failed => "workflow_failed",
3633        WorkflowTerminalKind::Cancelled => "cancelled",
3634        WorkflowTerminalKind::Terminated => "terminated",
3635        WorkflowTerminalKind::TimedOut => "timed_out",
3636    };
3637    let failure = description
3638        .failure
3639        .as_ref()
3640        .filter(|value| value.is_object());
3641    let nested_failure = failure
3642        .and_then(|value| value.get("failures"))
3643        .and_then(Value::as_array)
3644        .and_then(|failures| failures.last())
3645        .or_else(|| description.failures.last());
3646    let exception = description
3647        .exception
3648        .clone()
3649        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3650        .or_else(|| {
3651            nested_failure
3652                .and_then(|value| value.get("exception_payload"))
3653                .cloned()
3654        });
3655    let string_field = |name: &str| {
3656        failure
3657            .and_then(|value| value.get(name))
3658            .and_then(Value::as_str)
3659            .or_else(|| {
3660                nested_failure
3661                    .and_then(|value| value.get(name))
3662                    .and_then(Value::as_str)
3663            })
3664            .map(str::to_string)
3665    };
3666    let exception_field = |name: &str| {
3667        exception
3668            .as_ref()
3669            .and_then(|value| value.get(name))
3670            .and_then(Value::as_str)
3671            .map(str::to_string)
3672    };
3673    let message = description
3674        .error
3675        .clone()
3676        .or_else(|| string_field("message"))
3677        .or_else(|| exception_field("message"));
3678    let reason = description
3679        .raw
3680        .get("reason")
3681        .and_then(Value::as_str)
3682        .map(str::to_string)
3683        .or_else(|| {
3684            failure
3685                .and_then(|value| value.get("reason"))
3686                .and_then(Value::as_str)
3687                .map(str::to_string)
3688        })
3689        .or_else(|| description.closed_reason.clone())
3690        .unwrap_or_else(|| default_reason.to_string());
3691    let failure_id = string_field("failure_id").or_else(|| {
3692        nested_failure
3693            .and_then(|value| value.get("id"))
3694            .and_then(Value::as_str)
3695            .map(str::to_string)
3696    });
3697
3698    WorkflowTerminalOutcome {
3699        kind,
3700        workflow_id: description
3701            .workflow_id
3702            .clone()
3703            .unwrap_or_else(|| workflow_id.to_string()),
3704        run_id: description
3705            .run_id
3706            .clone()
3707            .or_else(|| run_id.map(str::to_string)),
3708        reason,
3709        failure_category: string_field("failure_category")
3710            .or_else(|| Some(default_reason.to_string())),
3711        failure_id,
3712        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3713        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3714        non_retryable: failure
3715            .and_then(|value| value.get("non_retryable"))
3716            .and_then(Value::as_bool)
3717            .or_else(|| {
3718                nested_failure
3719                    .and_then(|value| value.get("non_retryable"))
3720                    .and_then(Value::as_bool)
3721            }),
3722        message,
3723        exception,
3724        raw: description.raw_value(),
3725    }
3726}
3727
3728#[derive(Clone, Debug, Deserialize)]
3729pub struct RegisterWorkerResponse {
3730    pub worker_id: String,
3731    pub registered: bool,
3732    #[serde(default)]
3733    pub heartbeat_interval_seconds: Option<u64>,
3734    #[serde(default)]
3735    pub protocol_version: Option<String>,
3736    #[serde(default)]
3737    pub server_capabilities: Option<Value>,
3738}
3739
3740/// Result of gracefully removing a worker-plane registration.
3741#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3742pub struct WorkerDeregistrationEnvelope {
3743    pub worker_id: String,
3744    pub outcome: String,
3745    pub recovered_workflow_task_count: u64,
3746}
3747
3748#[derive(Clone, Debug, Deserialize)]
3749pub struct PollWorkflowTaskResponse {
3750    #[serde(default)]
3751    pub task: Option<WorkflowTask>,
3752    #[serde(default)]
3753    pub poll_status: Option<String>,
3754    #[serde(default)]
3755    pub reason: Option<String>,
3756    #[serde(default)]
3757    pub protocol_version: Option<String>,
3758    #[serde(default)]
3759    pub server_capabilities: Option<Value>,
3760}
3761
3762impl PollWorkflowTaskResponse {
3763    /// Classify this response without parsing server display text.
3764    pub fn outcome(&self) -> WorkerPollOutcome {
3765        worker_poll_outcome(
3766            self.task.is_some(),
3767            self.poll_status.as_deref(),
3768            self.reason.as_deref(),
3769        )
3770    }
3771}
3772
3773#[derive(Clone, Debug, Deserialize)]
3774pub struct PollActivityTaskResponse {
3775    #[serde(default)]
3776    pub task: Option<ActivityTask>,
3777    #[serde(default)]
3778    pub poll_status: Option<String>,
3779    #[serde(default)]
3780    pub reason: Option<String>,
3781}
3782
3783impl PollActivityTaskResponse {
3784    /// Classify this response without parsing server display text.
3785    pub fn outcome(&self) -> WorkerPollOutcome {
3786        worker_poll_outcome(
3787            self.task.is_some(),
3788            self.poll_status.as_deref(),
3789            self.reason.as_deref(),
3790        )
3791    }
3792}
3793
3794#[derive(Clone, Debug, Deserialize)]
3795pub struct PollQueryTaskResponse {
3796    #[serde(default)]
3797    pub task: Option<QueryTask>,
3798    #[serde(default)]
3799    pub poll_status: Option<String>,
3800    #[serde(default)]
3801    pub reason: Option<String>,
3802}
3803
3804impl PollQueryTaskResponse {
3805    /// Classify this response without parsing server display text.
3806    pub fn outcome(&self) -> WorkerPollOutcome {
3807        worker_poll_outcome(
3808            self.task.is_some(),
3809            self.poll_status.as_deref(),
3810            self.reason.as_deref(),
3811        )
3812    }
3813}
3814
3815/// Stable classification for worker poll responses.
3816#[derive(Clone, Debug, PartialEq, Eq)]
3817pub enum WorkerPollOutcome {
3818    /// A task was leased and is available on the response.
3819    Task,
3820    /// No task was leased, but the worker should continue polling.
3821    Idle {
3822        poll_status: Option<String>,
3823        reason: Option<String>,
3824    },
3825    /// The server asked this worker to stop claiming new work.
3826    Stop {
3827        poll_status: Option<String>,
3828        reason: Option<String>,
3829    },
3830}
3831
3832impl WorkerPollOutcome {
3833    pub fn should_stop(&self) -> bool {
3834        matches!(self, Self::Stop { .. })
3835    }
3836}
3837
3838fn worker_poll_outcome(
3839    has_task: bool,
3840    poll_status: Option<&str>,
3841    reason: Option<&str>,
3842) -> WorkerPollOutcome {
3843    if worker_poll_is_stop(poll_status, reason) {
3844        return WorkerPollOutcome::Stop {
3845            poll_status: poll_status.map(str::to_string),
3846            reason: reason.map(str::to_string),
3847        };
3848    }
3849
3850    if has_task {
3851        WorkerPollOutcome::Task
3852    } else {
3853        WorkerPollOutcome::Idle {
3854            poll_status: poll_status.map(str::to_string),
3855            reason: reason.map(str::to_string),
3856        }
3857    }
3858}
3859
3860/// An ephemeral server-routed query task.
3861#[derive(Clone, Debug, Deserialize)]
3862pub struct QueryTask {
3863    pub query_task_id: String,
3864    #[serde(default = "default_workflow_task_attempt")]
3865    pub query_task_attempt: u64,
3866    #[serde(default)]
3867    pub lease_owner: Option<String>,
3868    #[serde(default)]
3869    pub workflow_id: Option<String>,
3870    #[serde(default)]
3871    pub run_id: Option<String>,
3872    pub workflow_type: String,
3873    pub query_name: String,
3874    #[serde(
3875        default = "missing_task_payload_codec",
3876        deserialize_with = "deserialize_task_payload_codec"
3877    )]
3878    pub payload_codec: String,
3879    #[serde(default)]
3880    pub workflow_arguments: Option<Value>,
3881    #[serde(default)]
3882    pub query_arguments: Option<Value>,
3883    #[serde(default)]
3884    pub history_events: Vec<HistoryEvent>,
3885    #[serde(default)]
3886    pub history_export: Option<Value>,
3887    #[serde(default)]
3888    pub run_status: Option<String>,
3889}
3890
3891#[derive(Clone, Debug, Deserialize)]
3892pub struct WorkflowTask {
3893    pub task_id: String,
3894    #[serde(default)]
3895    pub workflow_id: Option<String>,
3896    #[serde(default)]
3897    pub run_id: Option<String>,
3898    pub workflow_type: String,
3899    #[serde(
3900        default = "missing_task_payload_codec",
3901        deserialize_with = "deserialize_task_payload_codec"
3902    )]
3903    pub payload_codec: String,
3904    #[serde(default)]
3905    pub arguments: Option<Value>,
3906    #[serde(default)]
3907    pub history_events: Vec<HistoryEvent>,
3908    #[serde(default)]
3909    pub total_history_events: Option<u64>,
3910    #[serde(default)]
3911    pub history_size_bytes: Option<u64>,
3912    #[serde(default)]
3913    pub continue_as_new_recommended: Option<bool>,
3914    #[serde(default)]
3915    pub history_budget_pressure: Option<String>,
3916    #[serde(default)]
3917    pub next_history_page_token: Option<String>,
3918    #[serde(default = "default_workflow_task_attempt")]
3919    pub workflow_task_attempt: u64,
3920    #[serde(default)]
3921    pub workflow_signal_id: Option<String>,
3922    #[serde(default)]
3923    pub signal_name: Option<String>,
3924    #[serde(default)]
3925    pub signal_arguments: Option<Value>,
3926    #[serde(default)]
3927    pub workflow_update_id: Option<String>,
3928    #[serde(default)]
3929    pub update_name: Option<String>,
3930    #[serde(default)]
3931    pub lease_owner: Option<String>,
3932}
3933
3934impl WorkflowTask {
3935    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3936        self.history_events.extend(page.history_events);
3937
3938        if page.total_history_events.is_some() {
3939            self.total_history_events = page.total_history_events;
3940        }
3941
3942        self.next_history_page_token = page
3943            .next_history_page_token
3944            .filter(|token| !token.is_empty());
3945    }
3946}
3947
3948#[derive(Clone, Debug, Deserialize)]
3949struct WorkflowTaskHistoryPage {
3950    #[serde(default)]
3951    history_events: Vec<HistoryEvent>,
3952    #[serde(default)]
3953    total_history_events: Option<u64>,
3954    #[serde(default)]
3955    next_history_page_token: Option<String>,
3956}
3957
3958#[derive(Clone, Debug, Deserialize)]
3959pub struct ActivityTask {
3960    pub task_id: String,
3961    #[serde(default)]
3962    pub activity_attempt_id: Option<String>,
3963    #[serde(default)]
3964    pub attempt_id: Option<String>,
3965    pub activity_type: String,
3966    #[serde(
3967        default = "missing_task_payload_codec",
3968        deserialize_with = "deserialize_task_payload_codec"
3969    )]
3970    pub payload_codec: String,
3971    #[serde(default)]
3972    pub arguments: Option<Value>,
3973    #[serde(default = "default_attempt_number")]
3974    pub attempt_number: u64,
3975    #[serde(default)]
3976    pub lease_owner: Option<String>,
3977}
3978
3979#[derive(Clone, Debug, Deserialize)]
3980pub struct HistoryEvent {
3981    #[serde(alias = "type")]
3982    pub event_type: String,
3983    #[serde(default)]
3984    pub payload: Value,
3985    #[serde(flatten)]
3986    pub raw: HashMap<String, Value>,
3987}
3988
3989/// One decoded signal in the committed workflow-history snapshot.
3990#[derive(Clone, Debug, PartialEq)]
3991pub struct QuerySignal {
3992    pub id: Option<String>,
3993    pub name: String,
3994    pub arguments: Vec<Value>,
3995    avro_arguments: Vec<AvroValue>,
3996    pub workflow_sequence: Option<u64>,
3997}
3998
3999impl QuerySignal {
4000    /// Lossless fixed Avro Value arguments for this committed signal.
4001    pub fn arguments_avro_value(&self) -> &[AvroValue] {
4002        &self.avro_arguments
4003    }
4004}
4005
4006/// Immutable state supplied to a registered query handler.
4007///
4008/// This context intentionally exposes no activity, signal-wait, or command
4009/// APIs. Query handlers inspect committed history and return a value; query
4010/// completion does not append an event or advance deterministic execution.
4011#[derive(Clone, Debug)]
4012pub struct QueryContext {
4013    pub workflow_id: Option<String>,
4014    pub run_id: Option<String>,
4015    pub workflow_type: String,
4016    pub run_status: Option<String>,
4017    workflow_input: Value,
4018    workflow_input_avro_value: AvroValue,
4019    history_events: Arc<Vec<HistoryEvent>>,
4020    signal_events: Arc<Vec<QuerySignal>>,
4021}
4022
4023impl QueryContext {
4024    /// The normalized argument list used to start the workflow.
4025    pub fn workflow_input(&self) -> &Value {
4026        &self.workflow_input
4027    }
4028
4029    /// The lossless fixed Avro Value argument list used to start the workflow.
4030    pub fn workflow_input_avro_value(&self) -> &AvroValue {
4031        &self.workflow_input_avro_value
4032    }
4033
4034    /// The immutable committed history used for this query snapshot.
4035    pub fn history_events(&self) -> &[HistoryEvent] {
4036        self.history_events.as_slice()
4037    }
4038
4039    /// All decoded signals in committed workflow order.
4040    pub fn signal_events(&self) -> &[QuerySignal] {
4041        self.signal_events.as_slice()
4042    }
4043
4044    /// Decoded argument lists for each committed signal with `signal_name`.
4045    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
4046        self.signal_events
4047            .iter()
4048            .filter(|signal| signal.name == signal_name)
4049            .map(|signal| signal.arguments.clone())
4050            .collect()
4051    }
4052
4053    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
4054    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
4055        self.signal_events
4056            .iter()
4057            .filter(|signal| signal.name == signal_name)
4058            .map(|signal| signal.avro_arguments.clone())
4059            .collect()
4060    }
4061}
4062
4063#[derive(Clone, Debug, Deserialize)]
4064pub struct ActivityHeartbeatResponse {
4065    #[serde(default)]
4066    pub cancel_requested: bool,
4067    #[serde(default)]
4068    pub heartbeat_recorded: bool,
4069    #[serde(default)]
4070    pub can_continue: Option<bool>,
4071    #[serde(default)]
4072    pub reason: Option<String>,
4073    #[serde(default)]
4074    pub run_closed_reason: Option<String>,
4075    #[serde(default)]
4076    pub run_closed_at: Option<String>,
4077    #[serde(default)]
4078    pub lease_expires_at: Option<String>,
4079    #[serde(default)]
4080    pub last_heartbeat_at: Option<String>,
4081}
4082
4083impl ActivityHeartbeatResponse {
4084    /// Whether the activity should stop instead of attempting completion.
4085    pub fn should_stop(&self) -> bool {
4086        self.cancel_requested || self.can_continue == Some(false)
4087    }
4088}
4089
4090fn missing_task_payload_codec() -> String {
4091    MISSING_TASK_PAYLOAD_CODEC.to_string()
4092}
4093
4094fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
4095where
4096    D: Deserializer<'de>,
4097{
4098    Ok(match Value::deserialize(deserializer)? {
4099        Value::String(codec) => codec,
4100        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
4101        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
4102    })
4103}
4104
4105fn default_workflow_task_attempt() -> u64 {
4106    1
4107}
4108
4109fn default_attempt_number() -> u64 {
4110    1
4111}
4112
4113type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4114type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
4115type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
4116type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
4117type ReplayedWorkflowHandler =
4118    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
4119type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4120type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
4121type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4122type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4123type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4124type ReplayedQueryHandler = Arc<
4125    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
4126        + Send
4127        + Sync,
4128>;
4129type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
4130
4131struct ReplayedWorkflowInvocation {
4132    future: WorkflowFuture,
4133    snapshot: WorkflowStateSnapshot,
4134}
4135
4136#[derive(Clone)]
4137struct RegisteredWorkflow {
4138    execute: WorkflowHandler,
4139    replay: Option<ReplayedWorkflowHandler>,
4140    state_type: Option<TypeId>,
4141}
4142
4143#[derive(Clone)]
4144enum RegisteredQuery {
4145    Snapshot(QueryHandler),
4146    Replayed {
4147        state_type: TypeId,
4148        handler: ReplayedQueryHandler,
4149    },
4150}
4151
4152#[derive(Clone, Debug)]
4153pub struct WorkerHeartbeatObservation {
4154    pub worker_id: String,
4155    pub task_queue: String,
4156    pub acknowledged_at_unix_millis: u64,
4157    pub acknowledgement: Value,
4158}
4159
4160/// Bounded retry policy for worker poll acquisition and worker heartbeats.
4161///
4162/// Expected empty long polls are normal successful responses. Transport
4163/// failures, HTTP 408/429 responses, and server errors are retried with capped
4164/// exponential backoff. Authentication, protocol, codec, and handler failures
4165/// are never retried by the worker.
4166#[derive(Clone, Copy, Debug)]
4167pub struct WorkerRetryPolicy {
4168    /// Number of retries after the initial request fails.
4169    pub max_retries: usize,
4170    /// Delay before the first retry.
4171    pub initial_backoff: Duration,
4172    /// Maximum delay between retries.
4173    pub max_backoff: Duration,
4174}
4175
4176impl Default for WorkerRetryPolicy {
4177    fn default() -> Self {
4178        Self {
4179            max_retries: 5,
4180            initial_backoff: Duration::from_millis(100),
4181            max_backoff: Duration::from_secs(5),
4182        }
4183    }
4184}
4185
4186#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4187enum ManagedPollOutcome {
4188    Idle,
4189    Handled,
4190    Stop,
4191}
4192
4193#[derive(Clone)]
4194pub struct Worker {
4195    client: Client,
4196    worker_id: String,
4197    task_queue: String,
4198    workflows: HashMap<String, RegisteredWorkflow>,
4199    activities: HashMap<String, ActivityHandler>,
4200    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4201    updates: HashMap<String, HashMap<String, UpdateHandler>>,
4202    max_concurrent_workflow_tasks: usize,
4203    max_concurrent_activity_tasks: usize,
4204    poll_timeout: Duration,
4205    heartbeat_interval: Duration,
4206    retry_policy: WorkerRetryPolicy,
4207    heartbeat_observer: Option<WorkerHeartbeatObserver>,
4208}
4209
4210impl Worker {
4211    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4212        Self {
4213            client,
4214            worker_id: default_worker_id(),
4215            task_queue: task_queue.into(),
4216            workflows: HashMap::new(),
4217            activities: HashMap::new(),
4218            queries: HashMap::new(),
4219            updates: HashMap::new(),
4220            max_concurrent_workflow_tasks: 10,
4221            max_concurrent_activity_tasks: 10,
4222            poll_timeout: Duration::from_secs(30),
4223            heartbeat_interval: Duration::from_secs(60),
4224            retry_policy: WorkerRetryPolicy::default(),
4225            heartbeat_observer: None,
4226        }
4227    }
4228
4229    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4230        self.worker_id = worker_id.into();
4231        self
4232    }
4233
4234    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4235        self.poll_timeout = timeout;
4236        self
4237    }
4238
4239    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4240        self.heartbeat_interval = interval;
4241        self
4242    }
4243
4244    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
4245    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4246        self.retry_policy = policy;
4247        self
4248    }
4249
4250    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4251    where
4252        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4253    {
4254        self.heartbeat_observer = Some(Arc::new(observer));
4255        self
4256    }
4257
4258    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4259        self.max_concurrent_workflow_tasks = count.max(1);
4260        self
4261    }
4262
4263    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4264        self.max_concurrent_activity_tasks = count.max(1);
4265        self
4266    }
4267
4268    /// Register a workflow handler.
4269    ///
4270    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
4271    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
4272    /// while acquiring or decoding a worker task remain worker-operation
4273    /// failures and do not get converted into workflow outcomes.
4274    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4275    where
4276        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4277        Fut: Future<Output = Result<Value>> + Send + 'static,
4278    {
4279        let handler = Arc::new(handler);
4280        self.workflows.insert(
4281            workflow_type.into(),
4282            RegisteredWorkflow {
4283                execute: Arc::new(move |ctx, input| {
4284                    let handler = Arc::clone(&handler);
4285                    Box::pin(async move {
4286                        let result = handler(ctx, input.into_json()?).await?;
4287                        AvroValue::from_serialize(&result)
4288                    })
4289                }),
4290                replay: None,
4291                state_type: None,
4292            },
4293        );
4294    }
4295
4296    /// Register a workflow on the lossless fixed Avro Value surface.
4297    pub fn register_workflow_avro_value<F, Fut>(
4298        &mut self,
4299        workflow_type: impl Into<String>,
4300        handler: F,
4301    ) where
4302        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4303        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4304    {
4305        self.workflows.insert(
4306            workflow_type.into(),
4307            RegisteredWorkflow {
4308                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4309                replay: None,
4310                state_type: None,
4311            },
4312        );
4313    }
4314
4315    /// Register a workflow whose typed instance state can be reconstructed for queries.
4316    ///
4317    /// `state_factory` creates a fresh instance for every normal workflow task and
4318    /// query replay. The workflow handler is the single source of truth for state
4319    /// transitions: it updates [`WorkflowInstance`] after activities and signals
4320    /// resolve. Query replay runs this same handler over committed history and
4321    /// discards any commands it would emit.
4322    pub fn register_replayed_workflow<S, Factory, F, Fut>(
4323        &mut self,
4324        workflow_type: impl Into<String>,
4325        state_factory: Factory,
4326        handler: F,
4327    ) where
4328        S: Clone + Send + Sync + 'static,
4329        Factory: Fn() -> S + Send + Sync + 'static,
4330        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4331        Fut: Future<Output = Result<Value>> + Send + 'static,
4332    {
4333        let state_factory = Arc::new(state_factory);
4334        let handler = Arc::new(handler);
4335
4336        let execute_factory = Arc::clone(&state_factory);
4337        let execute_handler = Arc::clone(&handler);
4338        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4339            let state = WorkflowInstance::new(execute_factory());
4340            let handler = Arc::clone(&execute_handler);
4341            Box::pin(async move {
4342                let result = handler(ctx, input.into_json()?, state).await?;
4343                AvroValue::from_serialize(&result)
4344            }) as WorkflowFuture
4345        });
4346
4347        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4348            let state = WorkflowInstance::new(state_factory());
4349            let snapshot_state = state.clone();
4350            let snapshot: WorkflowStateSnapshot =
4351                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4352            let replay_handler = Arc::clone(&handler);
4353            let future = async move {
4354                let result = replay_handler(ctx, input.into_json()?, state).await?;
4355                AvroValue::from_serialize(&result)
4356            };
4357            ReplayedWorkflowInvocation {
4358                future: Box::pin(future),
4359                snapshot,
4360            }
4361        });
4362
4363        self.workflows.insert(
4364            workflow_type.into(),
4365            RegisteredWorkflow {
4366                execute,
4367                replay: Some(replay),
4368                state_type: Some(TypeId::of::<S>()),
4369            },
4370        );
4371    }
4372
4373    /// Register a replayable workflow on the lossless fixed Avro Value surface.
4374    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4375        &mut self,
4376        workflow_type: impl Into<String>,
4377        state_factory: Factory,
4378        handler: F,
4379    ) where
4380        S: Clone + Send + Sync + 'static,
4381        Factory: Fn() -> S + Send + Sync + 'static,
4382        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4383        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4384    {
4385        let state_factory = Arc::new(state_factory);
4386        let handler = Arc::new(handler);
4387
4388        let execute_factory = Arc::clone(&state_factory);
4389        let execute_handler = Arc::clone(&handler);
4390        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4391            let state = WorkflowInstance::new(execute_factory());
4392            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4393        });
4394
4395        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4396            let state = WorkflowInstance::new(state_factory());
4397            let snapshot_state = state.clone();
4398            let snapshot: WorkflowStateSnapshot =
4399                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4400            ReplayedWorkflowInvocation {
4401                future: Box::pin(handler(ctx, input, state)),
4402                snapshot,
4403            }
4404        });
4405
4406        self.workflows.insert(
4407            workflow_type.into(),
4408            RegisteredWorkflow {
4409                execute,
4410                replay: Some(replay),
4411                state_type: Some(TypeId::of::<S>()),
4412            },
4413        );
4414    }
4415
4416    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4417    where
4418        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4419        Fut: Future<Output = Result<Value>> + Send + 'static,
4420    {
4421        let handler = Arc::new(handler);
4422        self.activities.insert(
4423            activity_type.into(),
4424            Arc::new(move |ctx, args| {
4425                let handler = Arc::clone(&handler);
4426                Box::pin(async move {
4427                    let result = handler(ctx, args.into_json()?).await?;
4428                    AvroValue::from_serialize(&result)
4429                })
4430            }),
4431        );
4432    }
4433
4434    /// Register an activity on the lossless fixed Avro Value surface.
4435    pub fn register_activity_avro_value<F, Fut>(
4436        &mut self,
4437        activity_type: impl Into<String>,
4438        handler: F,
4439    ) where
4440        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4441        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4442    {
4443        self.activities.insert(
4444            activity_type.into(),
4445            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4446        );
4447    }
4448
4449    /// Register a named, read-only query handler for a workflow type.
4450    ///
4451    /// The workflow type must also be registered with [`Worker::register_workflow`]
4452    /// before the worker runs. The handler receives only an immutable committed
4453    /// state snapshot and normalized query arguments.
4454    pub fn register_query<F, Fut>(
4455        &mut self,
4456        workflow_type: impl Into<String>,
4457        query_name: impl Into<String>,
4458        handler: F,
4459    ) where
4460        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4461        Fut: Future<Output = Result<Value>> + Send + 'static,
4462    {
4463        let handler = Arc::new(handler);
4464        self.queries
4465            .entry(workflow_type.into())
4466            .or_default()
4467            .insert(
4468                query_name.into(),
4469                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4470                    let handler = Arc::clone(&handler);
4471                    Box::pin(async move {
4472                        let result = handler(ctx, args.into_json()?).await?;
4473                        AvroValue::from_serialize(&result)
4474                    })
4475                })),
4476            );
4477    }
4478
4479    /// Register a query handler on the lossless fixed Avro Value surface.
4480    pub fn register_query_avro_value<F, Fut>(
4481        &mut self,
4482        workflow_type: impl Into<String>,
4483        query_name: impl Into<String>,
4484        handler: F,
4485    ) where
4486        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4487        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4488    {
4489        self.queries
4490            .entry(workflow_type.into())
4491            .or_default()
4492            .insert(
4493                query_name.into(),
4494                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4495            );
4496    }
4497
4498    /// Register a named query against deterministically replayed instance state.
4499    ///
4500    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
4501    /// same state type `S`. The handler receives an immutable, detached state
4502    /// clone, so successful and failed queries cannot affect workflow execution
4503    /// or the state reconstructed by a later query.
4504    pub fn register_replayed_query<S, F, Fut>(
4505        &mut self,
4506        workflow_type: impl Into<String>,
4507        query_name: impl Into<String>,
4508        handler: F,
4509    ) where
4510        S: Clone + Send + Sync + 'static,
4511        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4512        Fut: Future<Output = Result<Value>> + Send + 'static,
4513    {
4514        let handler = Arc::new(handler);
4515        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4516            let state = state.downcast::<S>().map_err(|_| {
4517                "registered query state type does not match the replayed workflow state".to_string()
4518            })?;
4519            let handler = Arc::clone(&handler);
4520            Ok(Box::pin(async move {
4521                let result = handler(ctx, state, args.into_json()?).await?;
4522                AvroValue::from_serialize(&result)
4523            }))
4524        });
4525
4526        self.queries
4527            .entry(workflow_type.into())
4528            .or_default()
4529            .insert(
4530                query_name.into(),
4531                RegisteredQuery::Replayed {
4532                    state_type: TypeId::of::<S>(),
4533                    handler: erased_handler,
4534                },
4535            );
4536    }
4537
4538    /// Register a replayed-state query on the lossless fixed Avro Value surface.
4539    pub fn register_replayed_query_avro_value<S, F, Fut>(
4540        &mut self,
4541        workflow_type: impl Into<String>,
4542        query_name: impl Into<String>,
4543        handler: F,
4544    ) where
4545        S: Clone + Send + Sync + 'static,
4546        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4547        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4548    {
4549        let handler = Arc::new(handler);
4550        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4551            let state = state.downcast::<S>().map_err(|_| {
4552                "registered query state type does not match the replayed workflow state".to_string()
4553            })?;
4554            Ok(Box::pin(handler(ctx, state, args)))
4555        });
4556
4557        self.queries
4558            .entry(workflow_type.into())
4559            .or_default()
4560            .insert(
4561                query_name.into(),
4562                RegisteredQuery::Replayed {
4563                    state_type: TypeId::of::<S>(),
4564                    handler: erased_handler,
4565                },
4566            );
4567    }
4568
4569    /// Register a synchronous workflow update handler.
4570    pub fn register_update<F, Fut>(
4571        &mut self,
4572        workflow_type: impl Into<String>,
4573        update_name: impl Into<String>,
4574        handler: F,
4575    ) where
4576        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4577        Fut: Future<Output = Result<Value>> + Send + 'static,
4578    {
4579        let handler = Arc::new(handler);
4580        self.updates
4581            .entry(workflow_type.into())
4582            .or_default()
4583            .insert(
4584                update_name.into(),
4585                Arc::new(move |ctx, args| {
4586                    let handler = Arc::clone(&handler);
4587                    Box::pin(async move {
4588                        let result = handler(ctx, args.into_json()?).await?;
4589                        AvroValue::from_serialize(&result)
4590                    })
4591                }),
4592            );
4593    }
4594
4595    /// Register an update handler on the lossless fixed Avro Value surface.
4596    pub fn register_update_avro_value<F, Fut>(
4597        &mut self,
4598        workflow_type: impl Into<String>,
4599        update_name: impl Into<String>,
4600        handler: F,
4601    ) where
4602        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4603        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4604    {
4605        self.updates
4606            .entry(workflow_type.into())
4607            .or_default()
4608            .insert(
4609                update_name.into(),
4610                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4611            );
4612    }
4613
4614    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4615        let mut command_contracts = serde_json::Map::new();
4616        for workflow_type in self.workflows.keys() {
4617            let mut queries = self
4618                .queries
4619                .get(workflow_type)
4620                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4621                .unwrap_or_default();
4622            queries.sort();
4623            let mut updates = self
4624                .updates
4625                .get(workflow_type)
4626                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4627                .unwrap_or_default();
4628            updates.sort();
4629            if !queries.is_empty() || !updates.is_empty() {
4630                command_contracts.insert(
4631                    workflow_type.clone(),
4632                    json!({
4633                        "queries": queries,
4634                        "updates": updates,
4635                        "update_validators": [],
4636                    }),
4637                );
4638            }
4639        }
4640
4641        self.client
4642            .register_worker_with_command_contracts(
4643                &self.worker_id,
4644                &self.task_queue,
4645                self.workflows.keys().cloned().collect(),
4646                self.activities.keys().cloned().collect(),
4647                self.max_concurrent_workflow_tasks,
4648                self.max_concurrent_activity_tasks,
4649                [
4650                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4651                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4652                ]
4653                .into_iter()
4654                .flatten()
4655                .collect(),
4656                Value::Object(command_contracts),
4657            )
4658            .await
4659    }
4660
4661    /// Run until shutdown or a terminal worker error occurs.
4662    ///
4663    /// Empty long-poll expirations do not stop the worker. Retryable poll and
4664    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
4665    /// authentication, protocol, and other non-retryable failures are returned.
4666    pub async fn run(&self) -> Result<()> {
4667        self.run_until(std::future::pending::<()>()).await
4668    }
4669
4670    /// Run until `shutdown` resolves or a terminal worker error occurs.
4671    ///
4672    /// This has the same liveness and terminal-error contract as [`Worker::run`].
4673    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4674    where
4675        F: Future<Output = ()>,
4676    {
4677        let registration = self.register().await?;
4678        if !registration.registered {
4679            return Err(Error::WorkerLoop(format!(
4680                "worker registration for {:?} was not accepted",
4681                self.worker_id
4682            )));
4683        }
4684        let registered_worker_id = registration.worker_id.clone();
4685        let primary = self.run_registered_until(shutdown, registration).await;
4686        let deregistration = self
4687            .client
4688            .deregister_worker_registration(&registered_worker_id)
4689            .await;
4690
4691        match (primary, deregistration) {
4692            (Ok(()), Ok(_)) => Ok(()),
4693            (Ok(()), Err(deregistration)) => Err(deregistration),
4694            (Err(primary), Ok(_)) => Err(primary),
4695            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4696                primary: Box::new(primary),
4697                deregistration: Box::new(deregistration),
4698            }),
4699        }
4700    }
4701
4702    async fn run_registered_until<F>(
4703        &self,
4704        shutdown: F,
4705        registration: RegisterWorkerResponse,
4706    ) -> Result<()>
4707    where
4708        F: Future<Output = ()>,
4709    {
4710        let heartbeat_interval = Duration::from_secs(
4711            registration
4712                .heartbeat_interval_seconds
4713                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4714        );
4715        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
4716        // from the completion of the preceding attempt, including its bounded
4717        // retries. A fixed-epoch interval can leave an already-due tick queued
4718        // while an acknowledgement is slow, producing a catch-up heartbeat as
4719        // soon as that request completes.
4720        let heartbeat = tokio::time::sleep(Duration::ZERO);
4721        tokio::pin!(heartbeat);
4722        tokio::pin!(shutdown);
4723        let stop = Arc::new(AtomicBool::new(false));
4724        // Poll responses may already have leased server-side work by the time
4725        // they become ready, so each poller owns its responses through
4726        // completion or failure instead of racing raw polls in this select.
4727        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4728            let worker = self.clone();
4729            let stop = Arc::clone(&stop);
4730            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4731        });
4732        let mut activity_poller = (!self.activities.is_empty()).then(|| {
4733            let worker = self.clone();
4734            let stop = Arc::clone(&stop);
4735            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4736        });
4737        let mut query_poller = (!self.queries.is_empty()).then(|| {
4738            let worker = self.clone();
4739            let stop = Arc::clone(&stop);
4740            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4741        });
4742
4743        loop {
4744            tokio::select! {
4745                _ = &mut shutdown => {
4746                    stop.store(true, Ordering::SeqCst);
4747                    break;
4748                }
4749                _ = &mut heartbeat => {
4750                    let result = self.retry_worker_operation(|| {
4751                        self.client.heartbeat_worker(
4752                            &self.worker_id,
4753                            self.max_concurrent_workflow_tasks,
4754                            self.max_concurrent_activity_tasks,
4755                        )
4756                    }).await;
4757                    heartbeat
4758                        .as_mut()
4759                        .reset(tokio::time::Instant::now() + heartbeat_interval);
4760                    match result {
4761                        Ok(acknowledgement) => {
4762                            if let Some(observer) = &self.heartbeat_observer {
4763                                observer(&WorkerHeartbeatObservation {
4764                                    worker_id: self.worker_id.clone(),
4765                                    task_queue: self.task_queue.clone(),
4766                                    acknowledged_at_unix_millis: SystemTime::now()
4767                                        .duration_since(UNIX_EPOCH)
4768                                        .unwrap_or_default()
4769                                        .as_millis()
4770                                        .min(u64::MAX as u128)
4771                                        as u64,
4772                                    acknowledgement,
4773                                });
4774                            }
4775                        }
4776                        Err(error) => {
4777                            stop.store(true, Ordering::SeqCst);
4778                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4779                            return Err(error);
4780                        }
4781                    }
4782                }
4783                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4784                    workflow_poller = None;
4785                    let stopped_by_server = stop.load(Ordering::SeqCst);
4786                    stop.store(true, Ordering::SeqCst);
4787                    let poller_result = optional_poller_result("workflow", result);
4788                    let join_result =
4789                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4790                    poller_result?;
4791                    join_result?;
4792                    if stopped_by_server {
4793                        return Ok(());
4794                    }
4795                    return Err(Error::WorkerLoop(
4796                        "workflow poller stopped unexpectedly".to_string(),
4797                    ));
4798                }
4799                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4800                    activity_poller = None;
4801                    let stopped_by_server = stop.load(Ordering::SeqCst);
4802                    stop.store(true, Ordering::SeqCst);
4803                    let poller_result = optional_poller_result("activity", result);
4804                    let join_result =
4805                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4806                    poller_result?;
4807                    join_result?;
4808                    if stopped_by_server {
4809                        return Ok(());
4810                    }
4811                    return Err(Error::WorkerLoop(
4812                        "activity poller stopped unexpectedly".to_string(),
4813                    ));
4814                }
4815                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4816                    query_poller = None;
4817                    let stopped_by_server = stop.load(Ordering::SeqCst);
4818                    stop.store(true, Ordering::SeqCst);
4819                    let poller_result = optional_poller_result("query", result);
4820                    let join_result =
4821                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4822                    poller_result?;
4823                    join_result?;
4824                    if stopped_by_server {
4825                        return Ok(());
4826                    }
4827                    return Err(Error::WorkerLoop(
4828                        "query poller stopped unexpectedly".to_string(),
4829                    ));
4830                }
4831            }
4832        }
4833
4834        join_pollers(
4835            workflow_poller.take(),
4836            activity_poller.take(),
4837            query_poller.take(),
4838        )
4839        .await
4840    }
4841
4842    /// Poll and settle at most one task from each enabled task family.
4843    ///
4844    /// A workflow may reach its server-enforced run deadline while this worker
4845    /// holds a task. When the completion endpoint authoritatively rejects that
4846    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
4847    /// terminal `run_status=failed`, the workflow tick is considered settled:
4848    /// the late command was not recorded and cannot replace the terminal run.
4849    /// Every other completion rejection remains an error. This worker-level
4850    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
4851    /// only bounds how long a client waits for a result.
4852    ///
4853    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
4854    /// the original [`Error::Http`] status and response body.
4855    pub async fn run_once(&self) -> Result<usize> {
4856        let mut handled = 0;
4857        match self.poll_workflow_once().await? {
4858            ManagedPollOutcome::Handled => handled += 1,
4859            ManagedPollOutcome::Stop => return Ok(handled),
4860            ManagedPollOutcome::Idle => {}
4861        }
4862        match self.poll_activity_once().await? {
4863            ManagedPollOutcome::Handled => handled += 1,
4864            ManagedPollOutcome::Stop => return Ok(handled),
4865            ManagedPollOutcome::Idle => {}
4866        }
4867        if !self.queries.is_empty() {
4868            match self.poll_query_once().await? {
4869                ManagedPollOutcome::Handled => handled += 1,
4870                ManagedPollOutcome::Stop => return Ok(handled),
4871                ManagedPollOutcome::Idle => {}
4872            }
4873        }
4874        Ok(handled)
4875    }
4876
4877    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4878        let poll_request_id = unique_request_id("rust-workflow-poll");
4879        let response = self
4880            .retry_worker_operation(|| {
4881                self.client.poll_workflow_task_response_with_request_id(
4882                    &self.worker_id,
4883                    &self.task_queue,
4884                    self.poll_timeout,
4885                    &poll_request_id,
4886                    0,
4887                )
4888            })
4889            .await?;
4890        if response.outcome().should_stop() {
4891            return Ok(ManagedPollOutcome::Stop);
4892        }
4893        let Some(task) = response.task else {
4894            return Ok(ManagedPollOutcome::Idle);
4895        };
4896
4897        let task_id = task.task_id.clone();
4898        let attempt = task.workflow_task_attempt;
4899        let run_id = task.run_id.clone();
4900        let lease_owner = task
4901            .lease_owner
4902            .clone()
4903            .unwrap_or_else(|| self.worker_id.clone());
4904
4905        match self.execute_workflow_task(task) {
4906            Ok(commands) if commands.is_empty() => {
4907                // A replay can consume a recorded pending durable command
4908                // without producing a new command. The standalone protocol
4909                // acknowledges that state through the typed waiting outcome;
4910                // an empty completion is rejected by servers that require at
4911                // least one executable command.
4912                self.client
4913                    .fail_workflow_task_with_type(
4914                        &task_id,
4915                        &lease_owner,
4916                        attempt,
4917                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4918                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4919                    )
4920                    .await?;
4921            }
4922            Ok(commands) => {
4923                let completion = self
4924                    .client
4925                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4926                    .await;
4927                if let Err(error) = completion {
4928                    if !workflow_task_completion_is_terminal_timeout(
4929                        &error,
4930                        &task_id,
4931                        attempt,
4932                        run_id.as_deref(),
4933                    ) {
4934                        return Err(error);
4935                    }
4936                }
4937            }
4938            Err(error) => {
4939                self.client
4940                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4941                    .await?;
4942            }
4943        }
4944
4945        Ok(ManagedPollOutcome::Handled)
4946    }
4947
4948    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4949        while !stop.load(Ordering::SeqCst) {
4950            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4951                stop.store(true, Ordering::SeqCst);
4952                break;
4953            }
4954        }
4955
4956        Ok(())
4957    }
4958
4959    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4960        let poll_request_id = unique_request_id("rust-activity-poll");
4961        let response = self
4962            .retry_worker_operation(|| {
4963                self.client.poll_activity_task_response_with_request_id(
4964                    &self.worker_id,
4965                    &self.task_queue,
4966                    self.poll_timeout,
4967                    &poll_request_id,
4968                    0,
4969                )
4970            })
4971            .await?;
4972        if response.outcome().should_stop() {
4973            return Ok(ManagedPollOutcome::Stop);
4974        }
4975        let Some(task) = response.task else {
4976            return Ok(ManagedPollOutcome::Idle);
4977        };
4978
4979        let task_id = task.task_id.clone();
4980        let attempt_id = task
4981            .activity_attempt_id
4982            .clone()
4983            .or(task.attempt_id.clone())
4984            .unwrap_or_default();
4985        let lease_owner = task
4986            .lease_owner
4987            .clone()
4988            .unwrap_or_else(|| self.worker_id.clone());
4989        let codec = task.payload_codec.clone();
4990        let result = self.execute_activity_task(task).await;
4991        match result {
4992            Ok(value) => {
4993                let completion = self
4994                    .client
4995                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4996                    .await;
4997                if let Err(error) = completion {
4998                    if !activity_task_rejection_is_final(&error) {
4999                        return Err(error);
5000                    }
5001                }
5002            }
5003            Err(error) => {
5004                let failure = self
5005                    .client
5006                    .fail_activity_task(
5007                        &task_id,
5008                        &attempt_id,
5009                        &lease_owner,
5010                        error.to_string(),
5011                        false,
5012                    )
5013                    .await;
5014                if let Err(error) = failure {
5015                    if !activity_task_rejection_is_final(&error) {
5016                        return Err(error);
5017                    }
5018                }
5019            }
5020        }
5021
5022        Ok(ManagedPollOutcome::Handled)
5023    }
5024
5025    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
5026        while !stop.load(Ordering::SeqCst) {
5027            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
5028                stop.store(true, Ordering::SeqCst);
5029                break;
5030            }
5031        }
5032
5033        Ok(())
5034    }
5035
5036    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
5037        let poll_request_id = unique_request_id("rust-query-poll");
5038        let response = self
5039            .retry_worker_operation(|| {
5040                self.client.poll_query_task_response_with_request_id(
5041                    &self.worker_id,
5042                    &self.task_queue,
5043                    self.poll_timeout,
5044                    &poll_request_id,
5045                    0,
5046                )
5047            })
5048            .await?;
5049        if response.outcome().should_stop() {
5050            return Ok(ManagedPollOutcome::Stop);
5051        }
5052        let Some(task) = response.task else {
5053            return Ok(ManagedPollOutcome::Idle);
5054        };
5055
5056        let query_task_id = task.query_task_id.clone();
5057        let attempt = task.query_task_attempt;
5058        let lease_owner = task
5059            .lease_owner
5060            .clone()
5061            .unwrap_or_else(|| self.worker_id.clone());
5062        let codec = task.payload_codec.clone();
5063
5064        match self.execute_query_task(task).await {
5065            Ok(value) => {
5066                let result_envelope = match encode_typed_envelope(&value, &codec) {
5067                    Ok(result_envelope) => result_envelope,
5068                    Err(error) => {
5069                        let failure = self
5070                            .client
5071                            .fail_query_task(
5072                                &query_task_id,
5073                                &lease_owner,
5074                                attempt,
5075                                error.to_string(),
5076                                "query_result_encode_failed",
5077                                "QueryResultEncodeFailed",
5078                            )
5079                            .await;
5080                        if let Err(error) = failure {
5081                            if !query_task_rejection_is_final(&error) {
5082                                return Err(error);
5083                            }
5084                        }
5085                        return Ok(ManagedPollOutcome::Handled);
5086                    }
5087                };
5088
5089                if let Err(error) = self
5090                    .client
5091                    .complete_query_task_with_envelope(
5092                        &query_task_id,
5093                        &lease_owner,
5094                        attempt,
5095                        value.clone().into_json()?,
5096                        result_envelope,
5097                    )
5098                    .await
5099                {
5100                    if !query_task_rejection_is_final(&error) {
5101                        return Err(error);
5102                    }
5103                }
5104            }
5105            Err(failure) => {
5106                let result = self
5107                    .client
5108                    .fail_query_task(
5109                        &query_task_id,
5110                        &lease_owner,
5111                        attempt,
5112                        failure.message,
5113                        failure.reason,
5114                        failure.failure_type,
5115                    )
5116                    .await;
5117                if let Err(error) = result {
5118                    if !query_task_rejection_is_final(&error) {
5119                        return Err(error);
5120                    }
5121                }
5122            }
5123        }
5124
5125        Ok(ManagedPollOutcome::Handled)
5126    }
5127
5128    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
5129        while !stop.load(Ordering::SeqCst) {
5130            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
5131                stop.store(true, Ordering::SeqCst);
5132                break;
5133            }
5134        }
5135
5136        Ok(())
5137    }
5138
5139    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5140    where
5141        F: FnMut() -> Fut,
5142        Fut: Future<Output = Result<T>>,
5143    {
5144        let mut retries = 0;
5145
5146        loop {
5147            match operation().await {
5148                Err(error)
5149                    if worker_operation_is_retryable(&error)
5150                        && retries < self.retry_policy.max_retries =>
5151                {
5152                    retries += 1;
5153                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5154                }
5155                result => return result,
5156            }
5157        }
5158    }
5159
5160    async fn execute_query_task(
5161        &self,
5162        mut task: QueryTask,
5163    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5164        validate_query_task_payloads(&task).map_err(|error| {
5165            QueryTaskExecutionFailure::new(
5166                "query_payload_decode_failed",
5167                error.to_string(),
5168                "QueryPayloadDecodeFailed",
5169            )
5170        })?;
5171
5172        if !self.workflows.contains_key(&task.workflow_type) {
5173            return Err(QueryTaskExecutionFailure::new(
5174                "query_workflow_type_not_registered",
5175                format!("no workflow registered for type {:?}", task.workflow_type),
5176                "WorkflowTypeNotRegistered",
5177            ));
5178        }
5179
5180        let Some(handlers) = self.queries.get(&task.workflow_type) else {
5181            return Err(QueryTaskExecutionFailure::new(
5182                "query_handler_unavailable",
5183                format!(
5184                    "query handlers are unavailable for workflow type {:?}",
5185                    task.workflow_type
5186                ),
5187                "QueryHandlerUnavailable",
5188            ));
5189        };
5190        let Some(query) = handlers.get(&task.query_name) else {
5191            return Err(QueryTaskExecutionFailure::new(
5192                "rejected_unknown_query",
5193                format!("unknown query {:?}", task.query_name),
5194                "QueryFailed",
5195            ));
5196        };
5197
5198        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5199            .map_err(|error| {
5200            QueryTaskExecutionFailure::new(
5201                "query_payload_decode_failed",
5202                format!("cannot decode query arguments: {error}"),
5203                "QueryPayloadDecodeFailed",
5204            )
5205        })?;
5206        let workflow_input_typed =
5207            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5208                .map_err(|error| {
5209                    QueryTaskExecutionFailure::new(
5210                        "query_workflow_state_unavailable",
5211                        format!("cannot decode workflow start input: {error}"),
5212                        "QueryWorkflowStateUnavailable",
5213                    )
5214                })?;
5215        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5216            QueryTaskExecutionFailure::new(
5217                "query_workflow_state_unavailable",
5218                format!("cannot project workflow start input: {error}"),
5219                "QueryWorkflowStateUnavailable",
5220            )
5221        })?;
5222        hydrate_query_history_from_export(&mut task).map_err(|error| {
5223            QueryTaskExecutionFailure::new(
5224                "query_workflow_state_unavailable",
5225                format!("cannot restore query history snapshot: {error}"),
5226                "QueryWorkflowStateUnavailable",
5227            )
5228        })?;
5229        enrich_query_history_from_export(&mut task).map_err(|error| {
5230            QueryTaskExecutionFailure::new(
5231                "query_workflow_state_unavailable",
5232                format!("cannot restore compact query history payloads: {error}"),
5233                "QueryWorkflowStateUnavailable",
5234            )
5235        })?;
5236        let signal_events = query_signal_events(&task).map_err(|error| {
5237            QueryTaskExecutionFailure::new(
5238                "query_workflow_state_unavailable",
5239                format!("cannot decode committed workflow signals: {error}"),
5240                "QueryWorkflowStateUnavailable",
5241            )
5242        })?;
5243        let history_events = Arc::new(std::mem::take(&mut task.history_events));
5244        let context = QueryContext {
5245            workflow_id: task.workflow_id,
5246            run_id: task.run_id,
5247            workflow_type: task.workflow_type.clone(),
5248            run_status: task.run_status,
5249            workflow_input,
5250            workflow_input_avro_value: workflow_input_typed.clone(),
5251            history_events: Arc::clone(&history_events),
5252            signal_events: Arc::new(signal_events),
5253        };
5254
5255        let future = match query {
5256            RegisteredQuery::Snapshot(handler) => handler(context, args),
5257            RegisteredQuery::Replayed {
5258                state_type,
5259                handler,
5260            } => {
5261                let workflow = self
5262                    .workflows
5263                    .get(&task.workflow_type)
5264                    .expect("workflow registration was checked above");
5265                if workflow.state_type != Some(*state_type) {
5266                    return Err(QueryTaskExecutionFailure::new(
5267                        "query_workflow_state_unavailable",
5268                        "replayed query state type does not match its workflow registration",
5269                        "QueryWorkflowStateUnavailable",
5270                    ));
5271                }
5272                let replay = workflow.replay.as_ref().ok_or_else(|| {
5273                    QueryTaskExecutionFailure::new(
5274                        "query_workflow_state_unavailable",
5275                        format!(
5276                            "workflow type {:?} is not registered for instance-state replay",
5277                            task.workflow_type
5278                        ),
5279                        "QueryWorkflowStateUnavailable",
5280                    )
5281                })?;
5282                let workflow_state = Arc::new(Mutex::new(
5283                    WorkflowState::new_with_identity(
5284                        history_events.as_ref().clone(),
5285                        context.workflow_id.clone(),
5286                        context.run_id.clone(),
5287                        self.task_queue.clone(),
5288                        task.payload_codec,
5289                        None,
5290                    )
5291                    .map_err(|error| {
5292                        QueryTaskExecutionFailure::new(
5293                            "query_workflow_state_unavailable",
5294                            format!("workflow replay failed before query: {error}"),
5295                            "QueryWorkflowStateUnavailable",
5296                        )
5297                    })?,
5298                ));
5299                let workflow_context = WorkflowContext {
5300                    state: workflow_state,
5301                };
5302                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5303                let mut cx = TaskContext::from_waker(noop_waker_ref());
5304                match invocation.future.as_mut().poll(&mut cx) {
5305                    Poll::Ready(Ok(_)) => {
5306                        workflow_context
5307                            .ensure_history_consumed()
5308                            .map_err(|error| {
5309                                QueryTaskExecutionFailure::new(
5310                                    "query_workflow_state_unavailable",
5311                                    format!("workflow replay failed before query: {error}"),
5312                                    "QueryWorkflowStateUnavailable",
5313                                )
5314                            })?;
5315                    }
5316                    Poll::Ready(Err(error)) => {
5317                        return Err(QueryTaskExecutionFailure::new(
5318                            "query_workflow_state_unavailable",
5319                            format!("workflow replay failed before query: {error}"),
5320                            "QueryWorkflowStateUnavailable",
5321                        ));
5322                    }
5323                    Poll::Pending => {
5324                        let commands = workflow_context.take_commands().map_err(|error| {
5325                            QueryTaskExecutionFailure::new(
5326                                "query_workflow_state_unavailable",
5327                                format!("workflow replay failed before query: {error}"),
5328                                "QueryWorkflowStateUnavailable",
5329                            )
5330                        })?;
5331                        if commands.is_empty()
5332                            && !workflow_context
5333                                .matched_recorded_pending()
5334                                .map_err(|error| {
5335                                    QueryTaskExecutionFailure::new(
5336                                        "query_workflow_state_unavailable",
5337                                        format!("workflow replay failed before query: {error}"),
5338                                        "QueryWorkflowStateUnavailable",
5339                                    )
5340                                })?
5341                        {
5342                            return Err(QueryTaskExecutionFailure::new(
5343                                "query_workflow_state_unavailable",
5344                                "workflow replay yielded without a durable command",
5345                                "QueryWorkflowStateUnavailable",
5346                            ));
5347                        }
5348                    }
5349                }
5350                let state = (invocation.snapshot)().map_err(|error| {
5351                    QueryTaskExecutionFailure::new(
5352                        "query_workflow_state_unavailable",
5353                        format!("cannot snapshot replayed workflow state: {error}"),
5354                        "QueryWorkflowStateUnavailable",
5355                    )
5356                })?;
5357                handler(context, state, args).map_err(|message| {
5358                    QueryTaskExecutionFailure::new(
5359                        "query_workflow_state_unavailable",
5360                        message,
5361                        "QueryWorkflowStateUnavailable",
5362                    )
5363                })?
5364            }
5365        };
5366
5367        future.await.map_err(|error| {
5368            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5369        })
5370    }
5371
5372    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5373        validate_workflow_task_payloads(&task)?;
5374
5375        if let Some(update_id) = task
5376            .workflow_update_id
5377            .as_deref()
5378            .filter(|update_id| !update_id.is_empty())
5379        {
5380            return self.execute_update_task(&task, update_id);
5381        }
5382
5383        let workflow = self
5384            .workflows
5385            .get(&task.workflow_type)
5386            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5387        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5388        let resume_signal = decode_resume_signal(&task)?;
5389        let history_budget = WorkflowHistoryBudget {
5390            event_count: task
5391                .total_history_events
5392                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5393            size_bytes: task.history_size_bytes,
5394            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5395            pressure: task.history_budget_pressure.clone(),
5396        };
5397        let mut workflow_state = WorkflowState::new_with_identity(
5398            task.history_events,
5399            task.workflow_id,
5400            task.run_id,
5401            self.task_queue.clone(),
5402            task.payload_codec.clone(),
5403            resume_signal,
5404        )?;
5405        workflow_state.history_budget = history_budget;
5406        let state = Arc::new(Mutex::new(workflow_state));
5407        let ctx = WorkflowContext { state };
5408        let mut future = (workflow.execute)(ctx.clone(), input);
5409        let mut cx = TaskContext::from_waker(noop_waker_ref());
5410
5411        match future.as_mut().poll(&mut cx) {
5412            Poll::Ready(Ok(result)) => {
5413                ctx.ensure_history_consumed()?;
5414                let result = encode_typed_envelope(&result, &task.payload_codec)?;
5415                let mut commands = ctx.take_commands()?;
5416                commands.push(json!({
5417                    "type": "complete_workflow",
5418                    "result": result
5419                }));
5420                Ok(commands)
5421            }
5422            Poll::Ready(Err(error)) => {
5423                if let Error::ContinueAsNew(request) = error {
5424                    let mut commands = ctx.take_commands()?;
5425                    if let Some(command) = ctx.continue_as_new_command(request)? {
5426                        commands.push(command);
5427                    }
5428                    ctx.ensure_history_consumed()?;
5429                    return Ok(commands);
5430                }
5431                // A handler error must not hide a committed durable command that
5432                // upgraded workflow code no longer consumes.
5433                ctx.ensure_history_consumed()?;
5434                if workflow_task_integrity_error(&error) {
5435                    // Replay and protocol failures describe the workflow-task
5436                    // decision itself. Do not let commands queued earlier in
5437                    // this uncommitted decision escape alongside a terminal
5438                    // workflow failure.
5439                    return Err(error);
5440                }
5441                let mut commands = ctx.take_commands()?;
5442                commands.push(workflow_failure_command(&error));
5443                Ok(commands)
5444            }
5445            Poll::Pending => {
5446                let commands = ctx.take_commands()?;
5447                if commands.is_empty() && !ctx.matched_recorded_pending()? {
5448                    Err(Error::WorkflowYieldedWithoutCommand)
5449                } else {
5450                    Ok(commands)
5451                }
5452            }
5453        }
5454    }
5455
5456    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5457        if !self.workflows.contains_key(&task.workflow_type) {
5458            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5459        }
5460
5461        let accepted = task.history_events.iter().rev().find_map(|event| {
5462            (event.event_type == "UpdateAccepted"
5463                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5464            .then_some(&event.payload)
5465        });
5466        let update_name = accepted
5467            .and_then(|payload| payload.get("update_name"))
5468            .and_then(Value::as_str)
5469            .or(task.update_name.as_deref())
5470            .unwrap_or_default();
5471        let Some(handler) = self
5472            .updates
5473            .get(&task.workflow_type)
5474            .and_then(|handlers| handlers.get(update_name))
5475        else {
5476            return Ok(vec![json!({
5477                "type": "fail_update",
5478                "update_id": update_id,
5479                "message": format!(
5480                    "no update handler is registered for {}.{update_name}",
5481                    task.workflow_type
5482                ),
5483                "exception_type": "UnknownUpdate",
5484                "non_retryable": true,
5485            })]);
5486        };
5487        let arguments = accepted
5488            .and_then(|payload| payload.get("arguments"))
5489            .or(task.arguments.as_ref());
5490        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5491        let context = QueryContext {
5492            workflow_id: task.workflow_id.clone(),
5493            run_id: task.run_id.clone(),
5494            workflow_type: task.workflow_type.clone(),
5495            run_status: Some("running".to_string()),
5496            workflow_input: Value::Null,
5497            workflow_input_avro_value: AvroValue::Null,
5498            history_events: Arc::new(task.history_events.clone()),
5499            signal_events: Arc::new(Vec::new()),
5500        };
5501        let mut future = handler(context, arguments);
5502        let mut cx = TaskContext::from_waker(noop_waker_ref());
5503
5504        match future.as_mut().poll(&mut cx) {
5505            Poll::Ready(Ok(result)) => Ok(vec![json!({
5506                "type": "complete_update",
5507                "update_id": update_id,
5508                "result": encode_typed_envelope(&result, &task.payload_codec)?,
5509            })]),
5510            Poll::Ready(Err(error)) => Ok(vec![json!({
5511                "type": "fail_update",
5512                "update_id": update_id,
5513                "message": error.to_string(),
5514                "exception_type": "UpdateFailed",
5515                "non_retryable": true,
5516            })]),
5517            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5518        }
5519    }
5520
5521    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5522        validate_activity_task_payloads(&task)?;
5523
5524        let handler = self
5525            .activities
5526            .get(&task.activity_type)
5527            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5528        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5529        let attempt_id = task
5530            .activity_attempt_id
5531            .clone()
5532            .or(task.attempt_id.clone())
5533            .unwrap_or_default();
5534        let lease_owner = task
5535            .lease_owner
5536            .clone()
5537            .unwrap_or_else(|| self.worker_id.clone());
5538        let ctx = ActivityContext {
5539            client: self.client.clone(),
5540            task_id: task.task_id,
5541            activity_attempt_id: attempt_id,
5542            lease_owner,
5543            activity_type: task.activity_type,
5544            attempt_number: task.attempt_number,
5545            task_queue: self.task_queue.clone(),
5546            worker_id: self.worker_id.clone(),
5547        };
5548
5549        handler(ctx, args).await
5550    }
5551}
5552
5553fn poller_result(
5554    kind: &str,
5555    result: std::result::Result<Result<()>, tokio::task::JoinError>,
5556) -> Result<()> {
5557    match result {
5558        Ok(result) => result,
5559        Err(error) => Err(Error::WorkerLoop(format!(
5560            "{kind} poller join error: {error}"
5561        ))),
5562    }
5563}
5564
5565fn optional_poller_result(
5566    kind: &str,
5567    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5568) -> Result<()> {
5569    match result {
5570        Some(result) => poller_result(kind, result),
5571        None => Ok(()),
5572    }
5573}
5574
5575async fn join_pollers(
5576    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5577    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5578    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5579) -> Result<()> {
5580    let mut first_error = None;
5581
5582    if let Some(handle) = workflow_poller {
5583        if let Err(error) = poller_result("workflow", handle.await) {
5584            first_error.get_or_insert(error);
5585        }
5586    }
5587
5588    if let Some(handle) = activity_poller {
5589        if let Err(error) = poller_result("activity", handle.await) {
5590            first_error.get_or_insert(error);
5591        }
5592    }
5593
5594    if let Some(handle) = query_poller {
5595        if let Err(error) = poller_result("query", handle.await) {
5596            first_error.get_or_insert(error);
5597        }
5598    }
5599
5600    if let Some(error) = first_error {
5601        Err(error)
5602    } else {
5603        Ok(())
5604    }
5605}
5606
5607fn default_worker_id() -> String {
5608    let millis = SystemTime::now()
5609        .duration_since(UNIX_EPOCH)
5610        .unwrap_or_default()
5611        .as_millis();
5612    format!("rust-worker-{}-{millis}", std::process::id())
5613}
5614
5615fn percent_encode_path_segment(segment: &str) -> String {
5616    const HEX: &[u8; 16] = b"0123456789ABCDEF";
5617    let mut encoded = String::with_capacity(segment.len());
5618
5619    for byte in segment.bytes() {
5620        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5621            encoded.push(char::from(byte));
5622        } else {
5623            encoded.push('%');
5624            encoded.push(char::from(HEX[(byte >> 4) as usize]));
5625            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5626        }
5627    }
5628
5629    encoded
5630}
5631
5632fn unique_request_id(prefix: &str) -> String {
5633    let nanos = SystemTime::now()
5634        .duration_since(UNIX_EPOCH)
5635        .unwrap_or_default()
5636        .as_nanos();
5637    format!("{prefix}-{}-{nanos}", std::process::id())
5638}
5639
5640#[derive(Debug)]
5641struct QueryTaskExecutionFailure {
5642    reason: String,
5643    message: String,
5644    failure_type: String,
5645}
5646
5647impl QueryTaskExecutionFailure {
5648    fn new(
5649        reason: impl Into<String>,
5650        message: impl Into<String>,
5651        failure_type: impl Into<String>,
5652    ) -> Self {
5653        Self {
5654            reason: reason.into(),
5655            message: message.into(),
5656            failure_type: failure_type.into(),
5657        }
5658    }
5659}
5660
5661/// Typed local state owned by one deterministic workflow invocation.
5662///
5663/// Use [`WorkflowInstance::update`] for the same state transitions during
5664/// ordinary execution and replay. A replayed query receives a detached
5665/// immutable `Arc<S>` rather than this mutation-capable handle.
5666#[derive(Clone, Debug)]
5667pub struct WorkflowInstance<S> {
5668    state: Arc<Mutex<S>>,
5669}
5670
5671impl<S> WorkflowInstance<S> {
5672    fn new(state: S) -> Self {
5673        Self {
5674            state: Arc::new(Mutex::new(state)),
5675        }
5676    }
5677
5678    /// Read the current workflow-instance state without changing it.
5679    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5680        let state = self
5681            .state
5682            .lock()
5683            .map_err(|_| Error::WorkflowStatePoisoned)?;
5684        Ok(reader(&state))
5685    }
5686
5687    /// Apply one deterministic workflow-instance state transition.
5688    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5689        let mut state = self
5690            .state
5691            .lock()
5692            .map_err(|_| Error::WorkflowStatePoisoned)?;
5693        Ok(transition(&mut state))
5694    }
5695}
5696
5697impl<S: Clone> WorkflowInstance<S> {
5698    fn snapshot(&self) -> Result<S> {
5699        self.read(Clone::clone)
5700    }
5701}
5702
5703#[derive(Clone, Debug)]
5704pub struct WorkflowContext {
5705    state: Arc<Mutex<WorkflowState>>,
5706}
5707
5708impl WorkflowContext {
5709    /// Identity of the parent workflow currently being replayed.
5710    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5711        let state = self
5712            .state
5713            .lock()
5714            .map_err(|_| Error::WorkflowStatePoisoned)?;
5715        Ok(WorkflowIdentity {
5716            workflow_id: state.workflow_id.clone(),
5717            run_id: state.run_id.clone(),
5718        })
5719    }
5720
5721    /// Return the server-published history budget for this workflow task.
5722    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5723        let state = self
5724            .state
5725            .lock()
5726            .map_err(|_| Error::WorkflowStatePoisoned)?;
5727        Ok(state.history_budget.clone())
5728    }
5729
5730    /// Continue this workflow instance as a fresh run with replacement arguments.
5731    ///
5732    /// Return this value directly from the workflow handler. The worker converts
5733    /// it to the terminal protocol command only after replay has consumed every
5734    /// recorded durable command.
5735    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5736        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5737    }
5738
5739    /// Continue as new with optional workflow-type and task-queue overrides.
5740    pub fn continue_as_new_with_options<T: Serialize>(
5741        &self,
5742        options: ContinueAsNewOptions,
5743        args: T,
5744    ) -> Result<Value> {
5745        options.validate()?;
5746        Err(Error::ContinueAsNew(ContinueAsNewRequest {
5747            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5748            options,
5749        }))
5750    }
5751
5752    pub fn activity<T: Serialize>(
5753        &self,
5754        activity_type: impl Into<String>,
5755        args: T,
5756    ) -> ActivityCall {
5757        self.activity_with_options(activity_type, ActivityOptions::new(), args)
5758    }
5759
5760    pub fn activity_on_queue<T, Q>(
5761        &self,
5762        activity_type: impl Into<String>,
5763        task_queue: Option<Q>,
5764        args: T,
5765    ) -> ActivityCall
5766    where
5767        T: Serialize,
5768        Q: Into<String>,
5769    {
5770        let mut options = ActivityOptions::new();
5771        options.task_queue = task_queue.map(Into::into);
5772        self.activity_with_options(activity_type, options, args)
5773    }
5774
5775    /// Schedule one durable activity with retry, routing, and timeout options.
5776    ///
5777    /// Options are validated before the command is emitted. Once the command is
5778    /// recorded, replay consumes the same activity lifecycle at this command
5779    /// position and never emits a duplicate schedule.
5780    ///
5781    /// ```no_run
5782    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
5783    /// # use std::time::Duration;
5784    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
5785    /// let result = ctx
5786    ///     .activity_with_options(
5787    ///         "charge-card",
5788    ///         ActivityOptions::new()
5789    ///             .task_queue("payments")
5790    ///             .retry_policy(
5791    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
5792    ///                     Duration::from_secs(1),
5793    ///                     2,
5794    ///                     Some(Duration::from_secs(30)),
5795    ///                 ),
5796    ///             )
5797    ///             .start_to_close_timeout(Duration::from_secs(60))
5798    ///             .schedule_to_close_timeout(Duration::from_secs(180))
5799    ///             .heartbeat_timeout(Duration::from_secs(15)),
5800    ///         json!([{"order_id": "order-42"}]),
5801    ///     )
5802    ///     .await;
5803    /// match result {
5804    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
5805    ///         "reason": failure.reason,
5806    ///         "timeout_kind": failure.timeout_kind,
5807    ///     })),
5808    ///     other => other,
5809    /// }
5810    /// # }
5811    /// ```
5812    pub fn activity_with_options<T: Serialize>(
5813        &self,
5814        activity_type: impl Into<String>,
5815        options: ActivityOptions,
5816        args: T,
5817    ) -> ActivityCall {
5818        ActivityCall {
5819            ctx: self.clone(),
5820            activity_type: activity_type.into(),
5821            options,
5822            args: Some(AvroValue::from_serialize(&args)),
5823            scheduled: false,
5824        }
5825    }
5826
5827    pub async fn activity_avro_value<T: Serialize>(
5828        &self,
5829        activity_type: impl Into<String>,
5830        args: T,
5831    ) -> Result<AvroValue> {
5832        let mut call = self.activity(activity_type, args);
5833        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5834    }
5835
5836    pub async fn activity_avro_value_with_options<T: Serialize>(
5837        &self,
5838        activity_type: impl Into<String>,
5839        options: ActivityOptions,
5840        args: T,
5841    ) -> Result<AvroValue> {
5842        let mut call = self.activity_with_options(activity_type, options, args);
5843        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5844    }
5845
5846    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5847        SignalCall {
5848            ctx: self.clone(),
5849            signal_name: signal_name.into(),
5850            opened_wait: false,
5851            matched_pending: false,
5852        }
5853    }
5854
5855    pub async fn wait_signal_avro_value(
5856        &self,
5857        signal_name: impl Into<String>,
5858    ) -> Result<Vec<AvroValue>> {
5859        let mut call = self.wait_signal(signal_name);
5860        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5861    }
5862
5863    /// Wait for server-backed durable time without blocking the worker executor.
5864    ///
5865    /// Polling this future emits one `start_timer` command and yields. The
5866    /// server records the deadline, so neither worker nor server restarts reset
5867    /// the wait. Replay resolves the future only from a `TimerScheduled` and
5868    /// `TimerFired` pair at the same position in the shared durable-command
5869    /// stream, with matching sequence, timer identity, and delay. Sub-second
5870    /// durations round up because protocol deadlines use whole seconds.
5871    ///
5872    /// ```no_run
5873    /// # use durable_workflow::{json, Client, Worker};
5874    /// # use std::time::Duration;
5875    /// # fn configure(client: Client) {
5876    /// let mut worker = Worker::new(client, "rust-workers");
5877    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
5878    ///     ctx.sleep(Duration::from_secs(5)).await?;
5879    ///     Ok(json!({"status": "timer fired"}))
5880    /// });
5881    /// # }
5882    /// ```
5883    pub fn sleep(&self, duration: Duration) -> TimerCall {
5884        let delay_seconds = duration
5885            .as_secs()
5886            .checked_add(u64::from(duration.subsec_nanos() > 0));
5887        TimerCall {
5888            ctx: self.clone(),
5889            delay_seconds,
5890            scheduled: false,
5891            matched_pending: false,
5892        }
5893    }
5894
5895    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
5896    pub fn start_timer(&self, duration: Duration) -> TimerCall {
5897        self.sleep(duration)
5898    }
5899
5900    /// Evaluate a non-deterministic callback once and durably record its typed value.
5901    ///
5902    /// On replay the callback is not invoked: the value is decoded from the
5903    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
5904    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
5905    /// small values that must remain fixed for the lifetime of a workflow run.
5906    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5907    where
5908        T: Serialize + DeserializeOwned,
5909        F: FnOnce() -> T,
5910    {
5911        {
5912            let mut state = self
5913                .state
5914                .lock()
5915                .map_err(|_| Error::WorkflowStatePoisoned)?;
5916            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5917                return match recorded {
5918                    RecordedCommand::SideEffect { sequence, value } => {
5919                        state.command_cursor += 1;
5920                        value.deserialize().map_err(|error| {
5921                            Error::NonDeterministicReplay(ReplayFailure::new(
5922                                "side_effect_type_mismatch",
5923                                Some(sequence),
5924                                Some(std::any::type_name::<T>().to_string()),
5925                                Some(error.to_string()),
5926                                "recorded side-effect value is incompatible with the requested Rust type",
5927                            ))
5928                        })
5929                    }
5930                    other => Err(command_mismatch(&other, "side effect")),
5931                };
5932            }
5933        }
5934
5935        let value = callback();
5936        let avro_value = AvroValue::from_serialize(&value)?;
5937        let mut state = self
5938            .state
5939            .lock()
5940            .map_err(|_| Error::WorkflowStatePoisoned)?;
5941        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5942        state.commands.push(json!({
5943            "type": "record_side_effect",
5944            "result": result,
5945        }));
5946        Ok(value)
5947    }
5948
5949    /// Record or replay a lossless fixed Avro Value side effect.
5950    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5951    where
5952        F: FnOnce() -> AvroValue,
5953    {
5954        {
5955            let mut state = self
5956                .state
5957                .lock()
5958                .map_err(|_| Error::WorkflowStatePoisoned)?;
5959            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5960                return match recorded {
5961                    RecordedCommand::SideEffect { value, .. } => {
5962                        state.command_cursor += 1;
5963                        Ok(value)
5964                    }
5965                    other => Err(command_mismatch(&other, "side effect")),
5966                };
5967            }
5968        }
5969
5970        let value = callback();
5971        let mut state = self
5972            .state
5973            .lock()
5974            .map_err(|_| Error::WorkflowStatePoisoned)?;
5975        let result = encode_typed_envelope(&value, &state.payload_codec)?;
5976        state.commands.push(json!({
5977            "type": "record_side_effect",
5978            "result": result,
5979        }));
5980        Ok(value)
5981    }
5982
5983    /// Record a UUIDv4 once and return the same UUID on every replay.
5984    pub fn uuid_v4(&self) -> Result<Uuid> {
5985        self.side_effect(Uuid::new_v4)
5986    }
5987
5988    /// Select the newest supported version for a change, or replay the version
5989    /// already committed for that stable change ID.
5990    pub fn get_version(
5991        &self,
5992        change_id: impl Into<String>,
5993        min_supported: i32,
5994        max_supported: i32,
5995    ) -> Result<i32> {
5996        let change_id = change_id.into();
5997        if change_id.trim().is_empty() {
5998            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5999                "version_change_id_invalid",
6000                None,
6001                Some("non-empty change ID".to_string()),
6002                Some(change_id),
6003                "version markers require a stable non-empty change ID",
6004            )));
6005        }
6006        if min_supported > max_supported {
6007            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6008                "version_range_invalid",
6009                None,
6010                Some("min_supported <= max_supported".to_string()),
6011                Some(format!("{min_supported}..={max_supported}")),
6012                "version marker supported range is invalid",
6013            )));
6014        }
6015
6016        let mut state = self
6017            .state
6018            .lock()
6019            .map_err(|_| Error::WorkflowStatePoisoned)?;
6020        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
6021            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
6022            return Ok(version);
6023        }
6024
6025        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6026            return match recorded {
6027                RecordedCommand::VersionMarker {
6028                    sequence,
6029                    change_id: recorded_change_id,
6030                    version,
6031                    ..
6032                } => {
6033                    if recorded_change_id != change_id {
6034                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6035                            "version_change_id_mismatch",
6036                            Some(sequence),
6037                            Some(recorded_change_id),
6038                            Some(change_id),
6039                            "recorded version marker change ID differs from current workflow code",
6040                        )));
6041                    }
6042                    ensure_version_supported(
6043                        &change_id,
6044                        version,
6045                        min_supported,
6046                        max_supported,
6047                        sequence,
6048                    )?;
6049                    state.command_cursor += 1;
6050                    state.version_markers.insert(change_id, (version, sequence));
6051                    Ok(version)
6052                }
6053                other => Err(command_mismatch(
6054                    &other,
6055                    format!("version marker:{change_id}"),
6056                )),
6057            };
6058        }
6059
6060        let version = max_supported;
6061        state.commands.push(json!({
6062            "type": "record_version_marker",
6063            "change_id": change_id,
6064            "version": version,
6065            "min_supported": min_supported,
6066            "max_supported": max_supported,
6067        }));
6068        // Sequence numbers are assigned by the server. Zero identifies a marker
6069        // selected in this uncommitted decision batch for duplicate-call checks.
6070        state.version_markers.insert(change_id, (version, 0));
6071        Ok(version)
6072    }
6073
6074    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
6075    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
6076        Ok(self.get_version(change_id, -1, 1)? == 1)
6077    }
6078
6079    /// Keep a patch marker in history after the legacy branch has been removed.
6080    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
6081        self.get_version(change_id, -1, 1).map(|_| ())
6082    }
6083
6084    /// Start a named durable child on an explicit queue and await its result.
6085    ///
6086    /// The command is recorded in the parent's sequence-ordered durable command
6087    /// stream. Replay keeps a scheduled child pending without emitting another
6088    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
6089    /// values preserve the history payload codec and include both sides of the
6090    /// durable relationship; failures are returned as
6091    /// [`Error::ChildWorkflowFailed`].
6092    ///
6093    /// ```no_run
6094    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
6095    /// # fn configure(client: Client) {
6096    /// let mut worker = Worker::new(client, "parent-workers");
6097    /// worker.register_workflow("order-parent", |ctx, _input| async move {
6098    ///     let child = ctx
6099    ///         .start_child_workflow(
6100    ///             "fulfil-order",
6101    ///             ChildWorkflowOptions::new("fulfilment-workers")
6102    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
6103    ///             json!([{"order_id": "order-42"}]),
6104    ///         )
6105    ///         .await?;
6106    ///     Ok(child.result)
6107    /// });
6108    /// # }
6109    /// ```
6110    pub fn start_child_workflow<T: Serialize>(
6111        &self,
6112        workflow_type: impl Into<String>,
6113        options: ChildWorkflowOptions,
6114        args: T,
6115    ) -> ChildWorkflowCall {
6116        ChildWorkflowCall {
6117            ctx: self.clone(),
6118            workflow_type: workflow_type.into(),
6119            options,
6120            args: Some(AvroValue::from_serialize(&args)),
6121            scheduled: false,
6122            matched_pending: false,
6123        }
6124    }
6125
6126    pub async fn start_child_workflow_avro_value<T: Serialize>(
6127        &self,
6128        workflow_type: impl Into<String>,
6129        options: ChildWorkflowOptions,
6130        args: T,
6131    ) -> Result<ChildWorkflowAvroResult> {
6132        let mut call = self.start_child_workflow(workflow_type, options, args);
6133        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6134    }
6135
6136    fn take_commands(&self) -> Result<Vec<Value>> {
6137        let mut state = self
6138            .state
6139            .lock()
6140            .map_err(|_| Error::WorkflowStatePoisoned)?;
6141        Ok(std::mem::take(&mut state.commands))
6142    }
6143
6144    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6145        let mut state = self
6146            .state
6147            .lock()
6148            .map_err(|_| Error::WorkflowStatePoisoned)?;
6149
6150        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6151            return Err(command_mismatch(&recorded, "continue as new"));
6152        }
6153        if state.recorded_continue_as_new_sequence.is_some() {
6154            state.continue_as_new_consumed = true;
6155            return Ok(None);
6156        }
6157
6158        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6159        let mut command = serde_json::Map::from_iter([
6160            ("type".to_string(), json!("continue_as_new")),
6161            ("arguments".to_string(), arguments),
6162            ("queue".to_string(), json!(state.task_queue.clone())),
6163        ]);
6164        if let Some(workflow_type) = request.options.workflow_type {
6165            command.insert("workflow_type".to_string(), json!(workflow_type));
6166        }
6167        if let Some(task_queue) = request.options.task_queue {
6168            command.insert("queue".to_string(), json!(task_queue));
6169        }
6170        Ok(Some(Value::Object(command)))
6171    }
6172
6173    fn matched_recorded_pending(&self) -> Result<bool> {
6174        let state = self
6175            .state
6176            .lock()
6177            .map_err(|_| Error::WorkflowStatePoisoned)?;
6178        Ok(state.matched_recorded_pending)
6179    }
6180
6181    fn ensure_history_consumed(&self) -> Result<()> {
6182        let state = self
6183            .state
6184            .lock()
6185            .map_err(|_| Error::WorkflowStatePoisoned)?;
6186        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6187            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6188                "recorded_commands_unconsumed",
6189                Some(command.sequence()),
6190                Some(command.shape().to_string()),
6191                Some("workflow completion".to_string()),
6192                "workflow completed before consuming all recorded durable commands",
6193            )));
6194        }
6195        if let Some(sequence) = state
6196            .recorded_continue_as_new_sequence
6197            .filter(|_| !state.continue_as_new_consumed)
6198        {
6199            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6200                "recorded_continue_as_new_unconsumed",
6201                Some(sequence),
6202                Some("continue as new".to_string()),
6203                Some("workflow completion".to_string()),
6204                "workflow completed without consuming its recorded continue-as-new transition",
6205            )));
6206        }
6207        Ok(())
6208    }
6209}
6210
6211#[derive(Debug)]
6212struct WorkflowState {
6213    workflow_id: Option<String>,
6214    run_id: Option<String>,
6215    task_queue: String,
6216    payload_codec: String,
6217    history_budget: WorkflowHistoryBudget,
6218    resume_signal: Option<ResumeSignal>,
6219    recorded_commands: Vec<RecordedCommand>,
6220    recorded_continue_as_new_sequence: Option<u64>,
6221    continue_as_new_consumed: bool,
6222    command_cursor: usize,
6223    matched_recorded_pending: bool,
6224    version_markers: HashMap<String, (i32, u64)>,
6225    commands: Vec<Value>,
6226}
6227
6228impl WorkflowState {
6229    #[cfg(test)]
6230    fn new(
6231        history: Vec<HistoryEvent>,
6232        task_queue: String,
6233        payload_codec: String,
6234        resume_signal: Option<ResumeSignal>,
6235    ) -> Result<Self> {
6236        Self::new_with_identity(
6237            history,
6238            None,
6239            None,
6240            task_queue,
6241            payload_codec,
6242            resume_signal,
6243        )
6244    }
6245
6246    fn new_with_identity(
6247        history: Vec<HistoryEvent>,
6248        workflow_id: Option<String>,
6249        run_id: Option<String>,
6250        task_queue: String,
6251        payload_codec: String,
6252        resume_signal: Option<ResumeSignal>,
6253    ) -> Result<Self> {
6254        let recorded_commands = recorded_commands(
6255            &history,
6256            &payload_codec,
6257            WorkflowIdentity {
6258                workflow_id: workflow_id.clone(),
6259                run_id: run_id.clone(),
6260            },
6261        )?;
6262        let recorded_continue_as_new = history
6263            .iter()
6264            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6265            .collect::<Vec<_>>();
6266        if recorded_continue_as_new.len() > 1 {
6267            return Err(invalid_recorded_history(
6268                "duplicate_continue_as_new_transition",
6269                recorded_continue_as_new
6270                    .last()
6271                    .and_then(|event| durable_event_sequence(event))
6272                    .unwrap_or(0),
6273                "one WorkflowContinuedAsNew event",
6274                &format!(
6275                    "{} WorkflowContinuedAsNew events",
6276                    recorded_continue_as_new.len()
6277                ),
6278                "workflow history records one continue-as-new transition more than once",
6279            ));
6280        }
6281        let recorded_continue_as_new_sequence = recorded_continue_as_new
6282            .first()
6283            .map(|event| {
6284                durable_event_sequence(event).ok_or_else(|| {
6285                    Error::NonDeterministicReplay(ReplayFailure::new(
6286                        "continue_as_new_sequence_missing",
6287                        None,
6288                        Some("recorded transition sequence".to_string()),
6289                        Some("missing sequence".to_string()),
6290                        "WorkflowContinuedAsNew history is missing its recorded sequence",
6291                    ))
6292                })
6293            })
6294            .transpose()?;
6295        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6296        Ok(Self {
6297            workflow_id,
6298            run_id,
6299            task_queue,
6300            payload_codec,
6301            history_budget: WorkflowHistoryBudget {
6302                event_count,
6303                ..WorkflowHistoryBudget::default()
6304            },
6305            resume_signal,
6306            recorded_commands,
6307            recorded_continue_as_new_sequence,
6308            continue_as_new_consumed: false,
6309            command_cursor: 0,
6310            matched_recorded_pending: false,
6311            version_markers: HashMap::new(),
6312            commands: Vec::new(),
6313        })
6314    }
6315}
6316
6317#[derive(Clone, Debug)]
6318enum RecordedCommand {
6319    Activity {
6320        sequence: u64,
6321        activity_type: Option<String>,
6322        options: Option<RecordedActivityOptions>,
6323        outcome: Option<ActivityOutcome>,
6324    },
6325    Timer {
6326        sequence: u64,
6327        delay_seconds: u64,
6328        fired: bool,
6329    },
6330    ChildWorkflow {
6331        sequence: u64,
6332        workflow_type: Option<String>,
6333        outcome: Option<ChildWorkflowOutcome>,
6334    },
6335    SignalWait {
6336        sequence: u64,
6337        signal_name: String,
6338        value: Option<Vec<AvroValue>>,
6339    },
6340    SideEffect {
6341        sequence: u64,
6342        value: AvroValue,
6343    },
6344    VersionMarker {
6345        sequence: u64,
6346        change_id: String,
6347        version: i32,
6348    },
6349}
6350
6351#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6352struct RecordedActivityOptions {
6353    task_queue: RecordedSnapshotValue<Option<String>>,
6354    execution_mode: RecordedSnapshotValue<Option<String>>,
6355    retry_policy: ActivityRetrySnapshot,
6356}
6357
6358#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6359enum RecordedSnapshotValue<T> {
6360    /// Older history did not persist this field, so it cannot constrain replay.
6361    Unknown,
6362    Known(T),
6363}
6364
6365impl<T: PartialEq> RecordedSnapshotValue<T> {
6366    fn matches_current(&self, current: &Self) -> bool {
6367        match self {
6368            Self::Unknown => true,
6369            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6370        }
6371    }
6372}
6373
6374#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6375struct ActivityRetrySnapshot {
6376    snapshot_version: RecordedSnapshotValue<Option<u64>>,
6377    max_attempts: RecordedSnapshotValue<Option<u64>>,
6378    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6379    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6380    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6381    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6382    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6383    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6384}
6385
6386impl ActivityRetrySnapshot {
6387    fn matches_current(&self, current: &Self) -> bool {
6388        self.snapshot_version
6389            .matches_current(&current.snapshot_version)
6390            && self.max_attempts.matches_current(&current.max_attempts)
6391            && self
6392                .backoff_seconds
6393                .matches_current(&current.backoff_seconds)
6394            && self
6395                .start_to_close_timeout
6396                .matches_current(&current.start_to_close_timeout)
6397            && self
6398                .schedule_to_start_timeout
6399                .matches_current(&current.schedule_to_start_timeout)
6400            && self
6401                .schedule_to_close_timeout
6402                .matches_current(&current.schedule_to_close_timeout)
6403            && self
6404                .heartbeat_timeout
6405                .matches_current(&current.heartbeat_timeout)
6406            && self
6407                .non_retryable_error_types
6408                .matches_current(&current.non_retryable_error_types)
6409    }
6410}
6411
6412fn recorded_optional_u64(
6413    object: Option<&serde_json::Map<String, Value>>,
6414    field: &str,
6415) -> RecordedSnapshotValue<Option<u64>> {
6416    match object.and_then(|object| object.get(field)) {
6417        None => RecordedSnapshotValue::Unknown,
6418        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6419        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6420    }
6421}
6422
6423fn recorded_optional_string(
6424    object: &serde_json::Map<String, Value>,
6425    field: &str,
6426) -> RecordedSnapshotValue<Option<String>> {
6427    match object.get(field) {
6428        None => RecordedSnapshotValue::Unknown,
6429        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6430        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6431    }
6432}
6433
6434fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6435    let policy = policy.and_then(Value::as_object);
6436    let backoff_seconds = policy
6437        .and_then(|policy| policy.get("backoff_seconds"))
6438        .and_then(Value::as_array)
6439        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6440        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6441    let mut non_retryable_error_types = Vec::new();
6442    for error_type in policy
6443        .and_then(|policy| policy.get("non_retryable_error_types"))
6444        .and_then(Value::as_array)
6445        .into_iter()
6446        .flatten()
6447        .filter_map(Value::as_str)
6448        .map(str::trim)
6449        .filter(|error_type| !error_type.is_empty())
6450    {
6451        if !non_retryable_error_types
6452            .iter()
6453            .any(|recorded| recorded == error_type)
6454        {
6455            non_retryable_error_types.push(error_type.to_string());
6456        }
6457    }
6458
6459    ActivityRetrySnapshot {
6460        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6461        max_attempts: recorded_optional_u64(policy, "max_attempts"),
6462        backoff_seconds,
6463        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6464        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6465        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6466        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6467        non_retryable_error_types: if policy
6468            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6469        {
6470            RecordedSnapshotValue::Known(non_retryable_error_types)
6471        } else {
6472            RecordedSnapshotValue::Unknown
6473        },
6474    }
6475}
6476
6477fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6478    let policy = options.retry_policy.as_ref();
6479    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6480        Some(Value::Null) => None,
6481        Some(value) => value_as_u64(value),
6482        None => Some(1),
6483    };
6484    let backoff_seconds = policy
6485        .and_then(|policy| policy.get("backoff_seconds"))
6486        .and_then(Value::as_array)
6487        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6488        .unwrap_or_default();
6489    let non_retryable_error_types = policy
6490        .and_then(|policy| policy.get("non_retryable_error_types"))
6491        .and_then(Value::as_array)
6492        .into_iter()
6493        .flatten()
6494        .filter_map(Value::as_str)
6495        .map(str::to_string)
6496        .collect();
6497
6498    ActivityRetrySnapshot {
6499        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6500        max_attempts: RecordedSnapshotValue::Known(max_attempts),
6501        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6502        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6503        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6504        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6505        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6506        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6507    }
6508}
6509
6510fn activity_options_description(options: &RecordedActivityOptions) -> String {
6511    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6512}
6513
6514impl RecordedCommand {
6515    fn sequence(&self) -> u64 {
6516        match self {
6517            Self::Activity { sequence, .. }
6518            | Self::Timer { sequence, .. }
6519            | Self::ChildWorkflow { sequence, .. }
6520            | Self::SignalWait { sequence, .. }
6521            | Self::SideEffect { sequence, .. }
6522            | Self::VersionMarker { sequence, .. } => *sequence,
6523        }
6524    }
6525
6526    fn shape(&self) -> &'static str {
6527        match self {
6528            Self::Activity { .. } => "activity",
6529            Self::Timer { .. } => "timer",
6530            Self::ChildWorkflow { .. } => "child workflow",
6531            Self::SignalWait { .. } => "signal wait",
6532            Self::SideEffect { .. } => "side effect",
6533            Self::VersionMarker { .. } => "version marker",
6534        }
6535    }
6536}
6537
6538fn ensure_version_supported(
6539    change_id: &str,
6540    version: i32,
6541    min_supported: i32,
6542    max_supported: i32,
6543    sequence: u64,
6544) -> Result<()> {
6545    if (min_supported..=max_supported).contains(&version) {
6546        return Ok(());
6547    }
6548    Err(Error::NonDeterministicReplay(ReplayFailure::new(
6549        "version_marker_incompatible_range",
6550        (sequence != 0).then_some(sequence),
6551        Some(format!("{min_supported}..={max_supported}")),
6552        Some(format!("{change_id}:{version}")),
6553        "recorded workflow version is outside the range supported by current code",
6554    )))
6555}
6556
6557#[derive(Clone, Debug)]
6558struct ResumeSignal {
6559    signal_name: String,
6560    arguments: Vec<AvroValue>,
6561}
6562
6563pub struct ActivityCall {
6564    ctx: WorkflowContext,
6565    activity_type: String,
6566    options: ActivityOptions,
6567    args: Option<Result<AvroValue>>,
6568    scheduled: bool,
6569}
6570
6571impl ActivityCall {
6572    fn poll_avro_value(
6573        mut self: Pin<&mut Self>,
6574        _cx: &mut TaskContext<'_>,
6575    ) -> Poll<Result<AvroValue>> {
6576        let ctx = self.ctx.clone();
6577        let mut state = match ctx.state.lock() {
6578            Ok(state) => state,
6579            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6580        };
6581
6582        if self.scheduled {
6583            return Poll::Pending;
6584        }
6585
6586        let options = match self.options.validate() {
6587            Ok(options) => options,
6588            Err(error) => {
6589                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6590            }
6591        };
6592        let task_queue = options
6593            .task_queue
6594            .clone()
6595            .unwrap_or_else(|| state.task_queue.clone());
6596        let current_recorded_options = RecordedActivityOptions {
6597            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6598            // Rust schedules ordinary durable activities. The server records a
6599            // non-null mode only for a specialized execution primitive.
6600            execution_mode: RecordedSnapshotValue::Known(None),
6601            retry_policy: current_activity_retry_snapshot(&options),
6602        };
6603
6604        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6605            let sequence = recorded.sequence();
6606            match recorded {
6607                RecordedCommand::Activity {
6608                    activity_type,
6609                    options: recorded_options,
6610                    outcome,
6611                    ..
6612                } => {
6613                    if let Some(recorded_type) = activity_type {
6614                        if recorded_type != self.activity_type {
6615                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6616                                ReplayFailure::new(
6617                                    "recorded_command_detail_mismatch",
6618                                    Some(sequence),
6619                                    Some(format!("activity:{recorded_type}")),
6620                                    Some(format!("activity:{}", self.activity_type)),
6621                                    "recorded activity type differs from the current workflow command",
6622                                ),
6623                            )));
6624                        }
6625                    }
6626                    if let Some(recorded_options) = recorded_options {
6627                        if !recorded_options
6628                            .task_queue
6629                            .matches_current(&current_recorded_options.task_queue)
6630                        {
6631                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6632                                ReplayFailure::new(
6633                                    "activity_task_queue_mismatch",
6634                                    Some(sequence),
6635                                    Some(activity_options_description(&recorded_options)),
6636                                    Some(activity_options_description(&current_recorded_options)),
6637                                    "recorded activity task queue differs from the current workflow command",
6638                                ),
6639                            )));
6640                        }
6641                        if !recorded_options
6642                            .execution_mode
6643                            .matches_current(&current_recorded_options.execution_mode)
6644                        {
6645                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6646                                ReplayFailure::new(
6647                                    "activity_execution_mode_mismatch",
6648                                    Some(sequence),
6649                                    Some(activity_options_description(&recorded_options)),
6650                                    Some(activity_options_description(&current_recorded_options)),
6651                                    "recorded activity execution mode differs from the current workflow command",
6652                                ),
6653                            )));
6654                        }
6655                        if !recorded_options
6656                            .retry_policy
6657                            .matches_current(&current_recorded_options.retry_policy)
6658                        {
6659                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6660                                ReplayFailure::new(
6661                                    "activity_retry_policy_mismatch",
6662                                    Some(sequence),
6663                                    Some(activity_options_description(&recorded_options)),
6664                                    Some(activity_options_description(&current_recorded_options)),
6665                                    "recorded activity retry policy differs from the current workflow command",
6666                                ),
6667                            )));
6668                        }
6669                    }
6670                    state.command_cursor += 1;
6671                    if let Some(outcome) = outcome {
6672                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6673                    }
6674                    state.matched_recorded_pending = true;
6675                    self.scheduled = true;
6676                    return Poll::Pending;
6677                }
6678                other => {
6679                    return Poll::Ready(Err(command_mismatch(
6680                        &other,
6681                        format!("activity:{}", self.activity_type),
6682                    )));
6683                }
6684            }
6685        }
6686
6687        if !self.scheduled {
6688            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6689                Ok(args) => args,
6690                Err(error) => return Poll::Ready(Err(error)),
6691            };
6692            let arguments = normalize_avro_arguments(args);
6693            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6694                Ok(envelope) => envelope,
6695                Err(error) => return Poll::Ready(Err(error)),
6696            };
6697
6698            let mut command = serde_json::Map::from_iter([
6699                ("type".to_string(), json!("schedule_activity")),
6700                (
6701                    "activity_type".to_string(),
6702                    json!(self.activity_type.clone()),
6703                ),
6704                ("queue".to_string(), json!(task_queue)),
6705                ("arguments".to_string(), envelope),
6706            ]);
6707            for (field, value) in [
6708                ("start_to_close_timeout", options.start_to_close_timeout),
6709                (
6710                    "schedule_to_start_timeout",
6711                    options.schedule_to_start_timeout,
6712                ),
6713                (
6714                    "schedule_to_close_timeout",
6715                    options.schedule_to_close_timeout,
6716                ),
6717                ("heartbeat_timeout", options.heartbeat_timeout),
6718            ] {
6719                if let Some(value) = value {
6720                    command.insert(field.to_string(), json!(value));
6721                }
6722            }
6723            if let Some(retry_policy) = options.retry_policy {
6724                command.insert("retry_policy".to_string(), retry_policy);
6725            }
6726            state.commands.push(Value::Object(command));
6727            self.scheduled = true;
6728        }
6729
6730        Poll::Pending
6731    }
6732}
6733
6734impl Future for ActivityCall {
6735    type Output = Result<Value>;
6736
6737    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6738        match self.poll_avro_value(cx) {
6739            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6740            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6741            Poll::Pending => Poll::Pending,
6742        }
6743    }
6744}
6745
6746/// Future returned by [`WorkflowContext::sleep`].
6747pub struct TimerCall {
6748    ctx: WorkflowContext,
6749    delay_seconds: Option<u64>,
6750    scheduled: bool,
6751    matched_pending: bool,
6752}
6753
6754impl Future for TimerCall {
6755    type Output = Result<()>;
6756
6757    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6758        if self.matched_pending {
6759            return Poll::Pending;
6760        }
6761
6762        let ctx = self.ctx.clone();
6763        let Some(requested_delay) = self.delay_seconds else {
6764            return Poll::Ready(Err(Error::TimerDurationOverflow));
6765        };
6766        let mut state = match ctx.state.lock() {
6767            Ok(state) => state,
6768            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6769        };
6770
6771        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6772            match recorded {
6773                RecordedCommand::Timer {
6774                    sequence,
6775                    delay_seconds,
6776                    fired,
6777                    ..
6778                } => {
6779                    if delay_seconds != requested_delay {
6780                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6781                            ReplayFailure::new(
6782                                "timer_delay_mismatch",
6783                                Some(sequence),
6784                                Some(format!("timer:{delay_seconds}s")),
6785                                Some(format!("timer:{requested_delay}s")),
6786                                "recorded timer delay differs from the current workflow command",
6787                            ),
6788                        )));
6789                    }
6790                    state.command_cursor += 1;
6791                    if fired {
6792                        return Poll::Ready(Ok(()));
6793                    }
6794                    state.matched_recorded_pending = true;
6795                    self.scheduled = true;
6796                    self.matched_pending = true;
6797                    return Poll::Pending;
6798                }
6799                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6800            }
6801        }
6802
6803        if !self.scheduled {
6804            state.commands.push(json!({
6805                "type": "start_timer",
6806                "delay_seconds": requested_delay,
6807            }));
6808            self.scheduled = true;
6809        }
6810
6811        Poll::Pending
6812    }
6813}
6814
6815/// Future returned by [`WorkflowContext::start_child_workflow`].
6816pub struct ChildWorkflowCall {
6817    ctx: WorkflowContext,
6818    workflow_type: String,
6819    options: ChildWorkflowOptions,
6820    args: Option<Result<AvroValue>>,
6821    scheduled: bool,
6822    matched_pending: bool,
6823}
6824
6825impl ChildWorkflowCall {
6826    fn poll_avro_value(
6827        mut self: Pin<&mut Self>,
6828        _cx: &mut TaskContext<'_>,
6829    ) -> Poll<Result<ChildWorkflowAvroResult>> {
6830        if self.matched_pending {
6831            return Poll::Pending;
6832        }
6833
6834        let ctx = self.ctx.clone();
6835        let mut state = match ctx.state.lock() {
6836            Ok(state) => state,
6837            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6838        };
6839
6840        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6841            let sequence = recorded.sequence();
6842            match recorded {
6843                RecordedCommand::ChildWorkflow {
6844                    workflow_type,
6845                    outcome,
6846                    ..
6847                } => {
6848                    if let Some(recorded_type) = workflow_type {
6849                        if recorded_type != self.workflow_type {
6850                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6851                                ReplayFailure::new(
6852                                    "recorded_command_detail_mismatch",
6853                                    Some(sequence),
6854                                    Some(format!("child workflow:{recorded_type}")),
6855                                    Some(format!("child workflow:{}", self.workflow_type)),
6856                                    "recorded child workflow type differs from the current workflow command",
6857                                ),
6858                            )));
6859                        }
6860                    }
6861                    state.command_cursor += 1;
6862                    if let Some(outcome) = outcome {
6863                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6864                    }
6865                    state.matched_recorded_pending = true;
6866                    self.scheduled = true;
6867                    self.matched_pending = true;
6868                    return Poll::Pending;
6869                }
6870                other => {
6871                    return Poll::Ready(Err(command_mismatch(
6872                        &other,
6873                        format!("child workflow:{}", self.workflow_type),
6874                    )));
6875                }
6876            }
6877        }
6878
6879        if !self.scheduled {
6880            if self.options.task_queue.trim().is_empty() {
6881                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6882                    "task_queue must not be empty".to_string(),
6883                )));
6884            }
6885            for (name, value) in [
6886                (
6887                    "execution_timeout_seconds",
6888                    self.options.execution_timeout_seconds,
6889                ),
6890                ("run_timeout_seconds", self.options.run_timeout_seconds),
6891            ] {
6892                if value == Some(0) {
6893                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6894                        "{name} must be at least 1"
6895                    ))));
6896                }
6897            }
6898
6899            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6900                Ok(args) => args,
6901                Err(error) => return Poll::Ready(Err(error)),
6902            };
6903            let arguments = match encode_typed_envelope(
6904                &normalize_avro_arguments(args),
6905                &state.payload_codec,
6906            ) {
6907                Ok(arguments) => arguments,
6908                Err(error) => return Poll::Ready(Err(error)),
6909            };
6910            let mut command = json!({
6911                "type": "start_child_workflow",
6912                "workflow_type": self.workflow_type,
6913                "queue": self.options.task_queue,
6914                "parent_close_policy": self.options.parent_close_policy.as_str(),
6915                "arguments": arguments,
6916            });
6917            let object = command
6918                .as_object_mut()
6919                .expect("child workflow command is always an object");
6920            if let Some(policy) = &self.options.retry_policy {
6921                let mut retry_policy = serde_json::Map::new();
6922                if let Some(max_attempts) = policy.max_attempts {
6923                    if max_attempts == 0 {
6924                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6925                            "retry_policy.max_attempts must be at least 1".to_string(),
6926                        )));
6927                    }
6928                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6929                }
6930                if !policy.backoff_seconds.is_empty() {
6931                    retry_policy
6932                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6933                }
6934                if !policy.non_retryable_error_types.is_empty() {
6935                    retry_policy.insert(
6936                        "non_retryable_error_types".to_string(),
6937                        json!(policy.non_retryable_error_types),
6938                    );
6939                }
6940                if retry_policy.is_empty() {
6941                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6942                        "retry_policy must configure at least one field".to_string(),
6943                    )));
6944                }
6945                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6946            }
6947            if let Some(seconds) = self.options.execution_timeout_seconds {
6948                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6949            }
6950            if let Some(seconds) = self.options.run_timeout_seconds {
6951                object.insert("run_timeout_seconds".to_string(), json!(seconds));
6952            }
6953            state.commands.push(command);
6954            self.scheduled = true;
6955        }
6956
6957        Poll::Pending
6958    }
6959}
6960
6961impl Future for ChildWorkflowCall {
6962    type Output = Result<ChildWorkflowResult>;
6963
6964    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6965        match self.poll_avro_value(cx) {
6966            Poll::Ready(Ok(result)) => match result.result.into_json() {
6967                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6968                    parent: result.parent,
6969                    child: result.child,
6970                    child_workflow_type: result.child_workflow_type,
6971                    result: projected,
6972                })),
6973                Err(error) => Poll::Ready(Err(error)),
6974            },
6975            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6976            Poll::Pending => Poll::Pending,
6977        }
6978    }
6979}
6980
6981fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6982    Error::NonDeterministicReplay(ReplayFailure::new(
6983        "recorded_command_mismatch",
6984        Some(recorded.sequence()),
6985        Some(recorded.shape().to_string()),
6986        Some(actual.into()),
6987        "current workflow command does not match the recorded durable command sequence",
6988    ))
6989}
6990
6991pub struct SignalCall {
6992    ctx: WorkflowContext,
6993    signal_name: String,
6994    opened_wait: bool,
6995    matched_pending: bool,
6996}
6997
6998impl SignalCall {
6999    fn poll_avro_value(
7000        mut self: Pin<&mut Self>,
7001        _cx: &mut TaskContext<'_>,
7002    ) -> Poll<Result<Vec<AvroValue>>> {
7003        if self.matched_pending {
7004            return Poll::Pending;
7005        }
7006
7007        let ctx = self.ctx.clone();
7008        let mut state = match ctx.state.lock() {
7009            Ok(state) => state,
7010            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
7011        };
7012
7013        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7014            match recorded {
7015                RecordedCommand::SignalWait {
7016                    sequence,
7017                    signal_name,
7018                    value,
7019                } => {
7020                    if signal_name != self.signal_name {
7021                        return Poll::Ready(Err(Error::NonDeterministicReplay(
7022                            ReplayFailure::new(
7023                                "recorded_command_detail_mismatch",
7024                                Some(sequence),
7025                                Some(format!("signal wait:{signal_name}")),
7026                                Some(format!("signal wait:{}", self.signal_name)),
7027                                "recorded signal name differs from the current workflow command",
7028                            ),
7029                        )));
7030                    }
7031
7032                    state.command_cursor += 1;
7033                    if let Some(value) = value {
7034                        return Poll::Ready(Ok(value));
7035                    }
7036                    if state
7037                        .resume_signal
7038                        .as_ref()
7039                        .is_some_and(|signal| signal.signal_name == self.signal_name)
7040                    {
7041                        let signal = state
7042                            .resume_signal
7043                            .take()
7044                            .expect("matching resume signal is present");
7045                        return Poll::Ready(Ok(signal.arguments));
7046                    }
7047
7048                    state.matched_recorded_pending = true;
7049                    self.opened_wait = true;
7050                    self.matched_pending = true;
7051                    return Poll::Pending;
7052                }
7053                other => {
7054                    return Poll::Ready(Err(command_mismatch(
7055                        &other,
7056                        format!("signal wait:{}", self.signal_name),
7057                    )));
7058                }
7059            }
7060        }
7061
7062        if state
7063            .resume_signal
7064            .as_ref()
7065            .is_some_and(|signal| signal.signal_name == self.signal_name)
7066        {
7067            let signal = state
7068                .resume_signal
7069                .take()
7070                .expect("matching resume signal is present");
7071            return Poll::Ready(Ok(signal.arguments));
7072        }
7073
7074        if !self.opened_wait {
7075            state.commands.push(json!({
7076                "type": "open_signal_wait",
7077                "signal_name": self.signal_name
7078            }));
7079            self.opened_wait = true;
7080        }
7081
7082        Poll::Pending
7083    }
7084}
7085
7086impl Future for SignalCall {
7087    type Output = Result<Vec<Value>>;
7088
7089    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
7090        match self.poll_avro_value(cx) {
7091            Poll::Ready(Ok(values)) => Poll::Ready(
7092                values
7093                    .into_iter()
7094                    .map(AvroValue::into_json)
7095                    .collect::<Result<Vec<_>>>(),
7096            ),
7097            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
7098            Poll::Pending => Poll::Pending,
7099        }
7100    }
7101}
7102
7103#[derive(Clone, Debug)]
7104pub struct ActivityContext {
7105    client: Client,
7106    pub task_id: String,
7107    pub activity_attempt_id: String,
7108    pub lease_owner: String,
7109    pub activity_type: String,
7110    pub attempt_number: u64,
7111    pub task_queue: String,
7112    pub worker_id: String,
7113}
7114
7115impl ActivityContext {
7116    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
7117        self.client
7118            .heartbeat_activity_task(
7119                &self.task_id,
7120                &self.activity_attempt_id,
7121                &self.lease_owner,
7122                details,
7123            )
7124            .await
7125    }
7126}
7127
7128fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
7129    validate_payload_codec(codec)?;
7130    match value {
7131        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
7132            value, codec,
7133        )?)),
7134        None => Ok(AvroValue::Array(Vec::new())),
7135    }
7136}
7137
7138fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7139    let Some(signal_name) = task
7140        .signal_name
7141        .as_deref()
7142        .filter(|value| !value.is_empty())
7143    else {
7144        return Ok(None);
7145    };
7146    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
7147    let AvroValue::Array(arguments) = decoded else {
7148        unreachable!("normalize_avro_arguments always returns an array");
7149    };
7150
7151    Ok(Some(ResumeSignal {
7152        signal_name: signal_name.to_string(),
7153        arguments,
7154    }))
7155}
7156
7157fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
7158    validate_payload_codec(&task.payload_codec)?;
7159    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
7160    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
7161    for event in &task.history_events {
7162        validate_history_event_payloads(event, &task.payload_codec)?;
7163    }
7164    Ok(())
7165}
7166
7167fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
7168    validate_payload_codec(&task.payload_codec)?;
7169    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
7170}
7171
7172fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
7173    validate_payload_codec(&task.payload_codec)?;
7174    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
7175    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
7176    for event in &task.history_events {
7177        validate_history_event_payloads(event, &task.payload_codec)?;
7178    }
7179
7180    let Some(export) = task.history_export.as_ref() else {
7181        return Ok(());
7182    };
7183    let export_codec = match export.get("payloads") {
7184        Some(payloads) => declared_payload_codec(payloads, "codec")?,
7185        None => None,
7186    }
7187    .unwrap_or(&task.payload_codec);
7188    validate_payload_codec(export_codec)?;
7189
7190    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
7191        for event in events {
7192            let event_type = event
7193                .get("event_type")
7194                .or_else(|| event.get("type"))
7195                .and_then(Value::as_str)
7196                .unwrap_or_default();
7197            if let Some(payload) = event.get("payload") {
7198                validate_history_payloads(event_type, payload, export_codec)?;
7199            }
7200        }
7201    }
7202    for signal in export
7203        .get("signals")
7204        .and_then(Value::as_array)
7205        .into_iter()
7206        .flatten()
7207    {
7208        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
7209        validate_payload_codec(codec)?;
7210        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
7211    }
7212    for activity in export
7213        .get("activities")
7214        .and_then(Value::as_array)
7215        .into_iter()
7216        .flatten()
7217    {
7218        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
7219        validate_payload_codec(codec)?;
7220        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
7221        validate_optional_inbound_payload(activity.get("result"), codec)?;
7222    }
7223    Ok(())
7224}
7225
7226fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
7227    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
7228}
7229
7230fn validate_history_payloads(
7231    event_type: &str,
7232    payload: &Value,
7233    fallback_codec: &str,
7234) -> Result<()> {
7235    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
7236    validate_payload_codec(codec)?;
7237    for field in history_payload_fields(event_type) {
7238        validate_optional_inbound_payload(payload.get(*field), codec)?;
7239    }
7240    Ok(())
7241}
7242
7243const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
7244
7245fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
7246    match event_type {
7247        "ActivityCompleted" => &["result"],
7248        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
7249        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
7250        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
7251        "ChildRunCompleted" => &["result", "output"],
7252        "WorkflowCompleted" => &["output"],
7253        "ServiceCallStarted"
7254        | "ServiceCallCompleted"
7255        | "ServiceCallFailed"
7256        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
7257        _ => &[],
7258    }
7259}
7260
7261fn signal_history_payload(payload: &Value) -> Option<&Value> {
7262    SIGNAL_HISTORY_PAYLOAD_FIELDS
7263        .iter()
7264        .find_map(|field| payload.get(*field))
7265}
7266
7267fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
7268    match value.get(field) {
7269        None => Ok(None),
7270        Some(Value::String(codec)) => Ok(Some(codec)),
7271        Some(_) => Err(invalid_payload_envelope()),
7272    }
7273}
7274
7275fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
7276    validate_payload_codec(codec)?;
7277    if let Some(value) = value.filter(|value| !value.is_null()) {
7278        decode_wire_avro_value(value, codec)?;
7279    }
7280    Ok(())
7281}
7282
7283fn recorded_commands(
7284    events: &[HistoryEvent],
7285    fallback_codec: &str,
7286    parent: WorkflowIdentity,
7287) -> Result<Vec<RecordedCommand>> {
7288    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7289    let mut last_new_sequence = None;
7290
7291    for event in events {
7292        let is_activity = matches!(
7293            event.event_type.as_str(),
7294            "ActivityScheduled"
7295                | "ActivityStarted"
7296                | "ActivityHeartbeatRecorded"
7297                | "ActivityRetryScheduled"
7298                | "ActivityCompleted"
7299                | "ActivityFailed"
7300                | "ActivityCancelled"
7301                | "ActivityTimedOut"
7302        );
7303        let is_workflow_timer = matches!(
7304            event.event_type.as_str(),
7305            "TimerScheduled" | "TimerCancelled" | "TimerFired"
7306        ) && !is_internal_timer_event(event);
7307        let is_child_workflow = matches!(
7308            event.event_type.as_str(),
7309            "ChildWorkflowScheduled"
7310                | "ChildRunCompleted"
7311                | "ChildRunFailed"
7312                | "ChildRunCancelled"
7313                | "ChildRunTerminated"
7314        );
7315        let is_signal_wait = is_recorded_signal_wait_event(event);
7316        let is_side_effect = event.event_type == "SideEffectRecorded";
7317        let is_version_marker = event.event_type == "VersionMarkerRecorded";
7318        if !is_activity
7319            && !is_workflow_timer
7320            && !is_child_workflow
7321            && !is_signal_wait
7322            && !is_side_effect
7323            && !is_version_marker
7324        {
7325            continue;
7326        }
7327
7328        let sequence = durable_event_sequence(event).ok_or_else(|| {
7329            Error::NonDeterministicReplay(ReplayFailure::new(
7330                "durable_command_sequence_missing",
7331                None,
7332                Some("positive workflow sequence".to_string()),
7333                Some(event.event_type.clone()),
7334                "durable command history event has no workflow sequence",
7335            ))
7336        })?;
7337        if sequence == 0 {
7338            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7339                "durable_command_sequence_invalid",
7340                Some(sequence),
7341                Some("positive workflow sequence".to_string()),
7342                Some(sequence.to_string()),
7343                "durable command history uses an invalid workflow sequence",
7344            )));
7345        }
7346        if !events_by_sequence.contains_key(&sequence) {
7347            if let Some(previous) = last_new_sequence {
7348                if sequence < previous {
7349                    return Err(invalid_recorded_history(
7350                        "durable_command_sequence_mismatch",
7351                        sequence,
7352                        &format!("workflow sequence greater than {previous}"),
7353                        &sequence.to_string(),
7354                        "durable commands are not strictly ordered by their recorded workflow sequence",
7355                    ));
7356                }
7357            }
7358            last_new_sequence = Some(sequence);
7359        }
7360        events_by_sequence.entry(sequence).or_default().push(event);
7361    }
7362
7363    let commands: Vec<RecordedCommand> = events_by_sequence
7364        .into_iter()
7365        .map(|(sequence, sequence_events)| {
7366            let activity_events: Vec<_> = sequence_events
7367                .iter()
7368                .copied()
7369                .filter(|event| event.event_type.starts_with("Activity"))
7370                .collect();
7371            let timer_events: Vec<_> = sequence_events
7372                .iter()
7373                .copied()
7374                .filter(|event| event.event_type.starts_with("Timer"))
7375                .collect();
7376            let child_events: Vec<_> = sequence_events
7377                .iter()
7378                .copied()
7379                .filter(|event| {
7380                    event.event_type == "ChildWorkflowScheduled"
7381                        || event.event_type.starts_with("ChildRun")
7382                })
7383                .collect();
7384            let signal_wait_events: Vec<_> = sequence_events
7385                .iter()
7386                .copied()
7387                .filter(|event| is_recorded_signal_wait_event(event))
7388                .collect();
7389            let side_effect_events: Vec<_> = sequence_events
7390                .iter()
7391                .copied()
7392                .filter(|event| event.event_type == "SideEffectRecorded")
7393                .collect();
7394            let version_marker_events: Vec<_> = sequence_events
7395                .iter()
7396                .copied()
7397                .filter(|event| event.event_type == "VersionMarkerRecorded")
7398                .collect();
7399
7400            let command_kind_count = usize::from(!activity_events.is_empty())
7401                + usize::from(!timer_events.is_empty())
7402                + usize::from(!child_events.is_empty())
7403                + usize::from(!signal_wait_events.is_empty())
7404                + usize::from(!side_effect_events.is_empty())
7405                + usize::from(!version_marker_events.is_empty());
7406            if command_kind_count > 1 {
7407                let actual = [
7408                    (!activity_events.is_empty()).then_some("activity"),
7409                    (!timer_events.is_empty()).then_some("timer"),
7410                    (!child_events.is_empty()).then_some("child workflow"),
7411                    (!signal_wait_events.is_empty()).then_some("signal wait"),
7412                    (!side_effect_events.is_empty()).then_some("side effect"),
7413                    (!version_marker_events.is_empty()).then_some("version marker"),
7414                ]
7415                .into_iter()
7416                .flatten()
7417                .collect::<Vec<_>>()
7418                .join(" and ");
7419                return Err(invalid_recorded_history(
7420                    "durable_command_sequence_collision",
7421                    sequence,
7422                    "one durable command kind",
7423                    &actual,
7424                    "one workflow sequence records more than one durable command kind",
7425                ));
7426            }
7427
7428            if !activity_events.is_empty() {
7429                let scheduled_count = activity_events
7430                    .iter()
7431                    .filter(|event| event.event_type == "ActivityScheduled")
7432                    .count();
7433                if scheduled_count > 1 {
7434                    return Err(invalid_recorded_history(
7435                        "duplicate_activity_schedule",
7436                        sequence,
7437                        "at most one ActivityScheduled event",
7438                        "multiple ActivityScheduled events",
7439                        "activity history schedules more than one command at one workflow sequence",
7440                    ));
7441                }
7442                let activity_type = activity_events.iter().find_map(|event| {
7443                    event
7444                        .payload
7445                        .get("activity_type")
7446                        .or_else(|| event.payload.get("activity_name"))
7447                        .and_then(Value::as_str)
7448                        .map(str::to_string)
7449                });
7450                if activity_events.iter().filter_map(|event| {
7451                    event
7452                        .payload
7453                        .get("activity_type")
7454                        .or_else(|| event.payload.get("activity_name"))
7455                        .and_then(Value::as_str)
7456                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7457                    return Err(invalid_recorded_history(
7458                        "activity_identity_mismatch",
7459                        sequence,
7460                        activity_type.as_deref().unwrap_or("one activity identity"),
7461                        "conflicting activity identities",
7462                        "activity lifecycle events at one workflow sequence disagree on identity",
7463                    ));
7464                }
7465                let terminal: Vec<_> = activity_events
7466                    .iter()
7467                    .copied()
7468                    .filter(|event| {
7469                        matches!(
7470                            event.event_type.as_str(),
7471                            "ActivityCompleted"
7472                                | "ActivityFailed"
7473                                | "ActivityCancelled"
7474                                | "ActivityTimedOut"
7475                        )
7476                    })
7477                    .collect();
7478                if terminal.len() > 1 {
7479                    return Err(invalid_recorded_history(
7480                        "duplicate_activity_terminal_event",
7481                        sequence,
7482                        "at most one terminal activity event",
7483                        "multiple terminal activity events",
7484                        "activity history settles one command more than once",
7485                    ));
7486                }
7487                let outcome = terminal
7488                    .first()
7489                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7490                    .transpose()?;
7491                let options = activity_events
7492                    .iter()
7493                    .find(|event| event.event_type == "ActivityScheduled")
7494                    .and_then(|event| event.payload.get("activity"))
7495                    .and_then(Value::as_object)
7496                    .map(|activity| RecordedActivityOptions {
7497                        task_queue: recorded_optional_string(activity, "queue"),
7498                        execution_mode: recorded_optional_string(activity, "execution_mode"),
7499                        retry_policy: recorded_activity_retry_snapshot(
7500                            activity.get("retry_policy"),
7501                        ),
7502                    });
7503                return Ok(RecordedCommand::Activity {
7504                    sequence,
7505                    activity_type,
7506                    options,
7507                    outcome,
7508                });
7509            }
7510
7511            if !child_events.is_empty() {
7512                let scheduled: Vec<_> = child_events
7513                    .iter()
7514                    .copied()
7515                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
7516                    .collect();
7517                if scheduled.len() != 1 {
7518                    return Err(invalid_recorded_history(
7519                        "child_workflow_schedule_missing_or_duplicate",
7520                        sequence,
7521                        "one ChildWorkflowScheduled event",
7522                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7523                        "child workflow replay requires exactly one recorded schedule event",
7524                    ));
7525                }
7526                let workflow_type = child_events.iter().find_map(|event| {
7527                    event
7528                        .payload
7529                        .get("child_workflow_type")
7530                        .or_else(|| event.payload.get("workflow_type"))
7531                        .and_then(Value::as_str)
7532                        .filter(|value| !value.is_empty())
7533                        .map(str::to_string)
7534                });
7535                if child_events
7536                    .iter()
7537                    .filter_map(|event| {
7538                        event
7539                            .payload
7540                            .get("child_workflow_type")
7541                            .or_else(|| event.payload.get("workflow_type"))
7542                            .and_then(Value::as_str)
7543                    })
7544                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
7545                {
7546                    return Err(invalid_recorded_history(
7547                        "child_workflow_identity_mismatch",
7548                        sequence,
7549                        workflow_type
7550                            .as_deref()
7551                            .unwrap_or("one child workflow type"),
7552                        "conflicting child workflow types",
7553                        "child workflow lifecycle events at one sequence disagree on type",
7554                    ));
7555                }
7556                let mut outcomes = child_workflow_outcomes(
7557                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7558                    fallback_codec,
7559                    parent.clone(),
7560                )?;
7561                if outcomes.len() > 1 {
7562                    return Err(invalid_recorded_history(
7563                        "duplicate_child_workflow_terminal_event",
7564                        sequence,
7565                        "at most one terminal child event",
7566                        "multiple terminal child events",
7567                        "child workflow history settles one command more than once",
7568                    ));
7569                }
7570                return Ok(RecordedCommand::ChildWorkflow {
7571                    sequence,
7572                    workflow_type,
7573                    outcome: outcomes.pop(),
7574                });
7575            }
7576
7577            if !signal_wait_events.is_empty() {
7578                let opened: Vec<_> = signal_wait_events
7579                    .iter()
7580                    .copied()
7581                    .filter(|event| event.event_type == "SignalWaitOpened")
7582                    .collect();
7583                if opened.len() != 1 {
7584                    return Err(invalid_recorded_history(
7585                        "signal_wait_open_missing_or_duplicate",
7586                        sequence,
7587                        "one SignalWaitOpened event",
7588                        &format!("{} SignalWaitOpened events", opened.len()),
7589                        "signal replay requires exactly one canonical wait-open event",
7590                    ));
7591                }
7592
7593                let applied: Vec<_> = signal_wait_events
7594                    .iter()
7595                    .copied()
7596                    .filter(|event| event.event_type == "SignalApplied")
7597                    .collect();
7598                if applied.len() > 1 {
7599                    return Err(invalid_recorded_history(
7600                        "duplicate_signal_wait_apply",
7601                        sequence,
7602                        "at most one SignalApplied event",
7603                        "multiple SignalApplied events",
7604                        "signal history applies one durable wait more than once",
7605                    ));
7606                }
7607
7608                let signal_names = signal_wait_events
7609                    .iter()
7610                    .map(|event| required_signal_wait_name(event, sequence))
7611                    .collect::<Result<Vec<_>>>()?;
7612                let signal_name = signal_names
7613                    .first()
7614                    .expect("signal wait events are not empty")
7615                    .clone();
7616                if signal_names.iter().any(|candidate| candidate != &signal_name) {
7617                    return Err(invalid_recorded_history(
7618                        "signal_wait_identity_mismatch",
7619                        sequence,
7620                        &signal_name,
7621                        "conflicting signal names",
7622                        "signal wait lifecycle events at one workflow sequence disagree on identity",
7623                    ));
7624                }
7625                let value = applied
7626                    .first()
7627                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
7628                    .transpose()?;
7629                return Ok(RecordedCommand::SignalWait {
7630                    sequence,
7631                    signal_name,
7632                    value,
7633                });
7634            }
7635
7636            if !side_effect_events.is_empty() {
7637                if side_effect_events.len() != 1 {
7638                    return Err(invalid_recorded_history(
7639                        "duplicate_side_effect_record",
7640                        sequence,
7641                        "one SideEffectRecorded event",
7642                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
7643                        "side-effect history records one workflow command more than once",
7644                    ));
7645                }
7646                let event = side_effect_events[0];
7647                let result = event.payload.get("result").ok_or_else(|| {
7648                    invalid_recorded_history(
7649                        "side_effect_result_missing",
7650                        sequence,
7651                        "recorded result payload",
7652                        "missing result",
7653                        "side-effect history is missing its recorded value",
7654                    )
7655                })?;
7656                let has_published_envelope = result.as_str().is_some()
7657                    || result.as_object().is_some_and(|envelope| {
7658                        envelope.get("codec").and_then(Value::as_str).is_some()
7659                            && envelope.get("blob").and_then(Value::as_str).is_some()
7660                    });
7661                if !has_published_envelope {
7662                    return Err(invalid_recorded_history(
7663                        "side_effect_payload_malformed",
7664                        sequence,
7665                        "payload blob or {codec, blob} envelope",
7666                        &result.to_string(),
7667                        "side-effect history result does not use a published payload envelope",
7668                    ));
7669                }
7670                let codec = event
7671                    .payload
7672                    .get("payload_codec")
7673                    .and_then(Value::as_str)
7674                    .unwrap_or(fallback_codec);
7675                let value = decode_wire_avro_value(result, codec).map_err(|error| {
7676                    if error.to_string().contains("unsupported_payload_codec") {
7677                        return error;
7678                    }
7679
7680                    invalid_recorded_history(
7681                        "side_effect_payload_incompatible",
7682                        sequence,
7683                        &format!("valid {codec} payload envelope"),
7684                        &error.to_string(),
7685                        "side-effect history payload cannot be decoded with its recorded codec",
7686                    )
7687                })?;
7688                return Ok(RecordedCommand::SideEffect { sequence, value });
7689            }
7690
7691            if !version_marker_events.is_empty() {
7692                if version_marker_events.len() != 1 {
7693                    return Err(invalid_recorded_history(
7694                        "duplicate_version_marker_record",
7695                        sequence,
7696                        "one VersionMarkerRecorded event",
7697                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7698                        "version-marker history records one workflow command more than once",
7699                    ));
7700                }
7701                let payload = &version_marker_events[0].payload;
7702                let change_id = payload
7703                    .get("change_id")
7704                    .and_then(Value::as_str)
7705                    .filter(|value| !value.is_empty())
7706                    .map(str::to_string)
7707                    .ok_or_else(|| {
7708                        invalid_recorded_history(
7709                            "version_marker_field_missing",
7710                            sequence,
7711                            "non-empty change_id",
7712                            "missing or invalid change_id",
7713                            "version-marker history is missing its stable change ID",
7714                        )
7715                    })?;
7716                let version = required_version_i32(payload, "version", sequence)?;
7717                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7718                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7719                if min_supported > max_supported || version < min_supported || version > max_supported {
7720                    return Err(invalid_recorded_history(
7721                        "version_marker_history_range_invalid",
7722                        sequence,
7723                        "min_supported <= version <= max_supported",
7724                        &format!("{min_supported} <= {version} <= {max_supported}"),
7725                        "recorded version marker contains an internally incompatible range",
7726                    ));
7727                }
7728                return Ok(RecordedCommand::VersionMarker {
7729                    sequence,
7730                    change_id,
7731                    version,
7732                });
7733            }
7734
7735            let scheduled: Vec<_> = timer_events
7736                .iter()
7737                .copied()
7738                .filter(|event| event.event_type == "TimerScheduled")
7739                .collect();
7740            let fired: Vec<_> = timer_events
7741                .iter()
7742                .copied()
7743                .filter(|event| event.event_type == "TimerFired")
7744                .collect();
7745            if scheduled.len() != 1 {
7746                return Err(invalid_recorded_history(
7747                    "timer_schedule_missing_or_duplicate",
7748                    sequence,
7749                    "one TimerScheduled event",
7750                    &format!("{} TimerScheduled events", scheduled.len()),
7751                    "timer replay requires exactly one recorded schedule event",
7752                ));
7753            }
7754            if fired.len() > 1 {
7755                return Err(invalid_recorded_history(
7756                    "duplicate_timer_fire",
7757                    sequence,
7758                    "at most one TimerFired event",
7759                    "multiple TimerFired events",
7760                    "timer history contains more than one fire event for a workflow sequence",
7761                ));
7762            }
7763
7764            let scheduled = scheduled[0];
7765            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7766            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7767            if let Some(fired) = fired.first() {
7768                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7769                if fired_timer_id != timer_id {
7770                    return Err(invalid_recorded_history(
7771                        "timer_identity_mismatch",
7772                        sequence,
7773                        &timer_id,
7774                        &fired_timer_id,
7775                        "TimerFired does not correspond to the recorded TimerScheduled event",
7776                    ));
7777                }
7778                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7779                if fired_delay != delay_seconds {
7780                    return Err(invalid_recorded_history(
7781                        "timer_history_delay_mismatch",
7782                        sequence,
7783                        &delay_seconds.to_string(),
7784                        &fired_delay.to_string(),
7785                        "TimerScheduled and TimerFired record different delays",
7786                    ));
7787                }
7788            }
7789
7790            Ok(RecordedCommand::Timer {
7791                sequence,
7792                delay_seconds,
7793                fired: !fired.is_empty(),
7794            })
7795        })
7796        .collect::<Result<_>>()?;
7797
7798    let mut marker_sequences = HashMap::new();
7799    for command in &commands {
7800        if let RecordedCommand::VersionMarker {
7801            sequence,
7802            change_id,
7803            ..
7804        } = command
7805        {
7806            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7807                return Err(invalid_recorded_history(
7808                    "duplicate_version_marker",
7809                    *sequence,
7810                    &format!("one marker for change ID {change_id:?}"),
7811                    &format!("markers at sequences {first_sequence} and {sequence}"),
7812                    "workflow history contains duplicate markers for one stable change ID",
7813                ));
7814            }
7815        }
7816    }
7817
7818    Ok(commands)
7819}
7820
7821fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7822    payload
7823        .get(field)
7824        .and_then(Value::as_i64)
7825        .and_then(|value| i32::try_from(value).ok())
7826        .ok_or_else(|| {
7827            invalid_recorded_history(
7828                "version_marker_field_missing",
7829                sequence,
7830                &format!("integer {field}"),
7831                "missing or out-of-range integer",
7832                "version-marker history is missing a required integer field",
7833            )
7834        })
7835}
7836
7837fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7838    event
7839        .payload
7840        .get("sequence")
7841        .or_else(|| event.payload.get("workflow_sequence"))
7842        .or_else(|| event.raw.get("sequence"))
7843        .or_else(|| event.raw.get("workflow_sequence"))
7844        .and_then(value_as_u64)
7845}
7846
7847fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7848    matches!(
7849        event
7850            .payload
7851            .get("timer_kind")
7852            .or_else(|| event.raw.get("timer_kind"))
7853            .and_then(Value::as_str),
7854        Some("condition_timeout" | "signal_timeout")
7855    )
7856}
7857
7858fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7859    event
7860        .payload
7861        .get("signal_name")
7862        .or_else(|| event.raw.get("signal_name"))
7863        .and_then(Value::as_str)
7864        .filter(|value| !value.is_empty())
7865        .map(str::to_string)
7866        .ok_or_else(|| {
7867            invalid_recorded_history(
7868                "signal_wait_name_missing",
7869                sequence,
7870                "non-empty signal_name",
7871                &event.event_type,
7872                "canonical signal-wait history is missing its signal identity",
7873            )
7874        })
7875}
7876
7877fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7878    matches!(
7879        event.event_type.as_str(),
7880        "SignalWaitOpened" | "SignalApplied"
7881    )
7882}
7883
7884fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7885    event
7886        .payload
7887        .get(field)
7888        .and_then(Value::as_str)
7889        .filter(|value| !value.is_empty())
7890        .map(str::to_string)
7891        .ok_or_else(|| {
7892            invalid_recorded_history(
7893                "timer_history_field_missing",
7894                sequence,
7895                field,
7896                &event.event_type,
7897                "timer history is missing a required identity field",
7898            )
7899        })
7900}
7901
7902fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7903    event
7904        .payload
7905        .get(field)
7906        .and_then(value_as_u64)
7907        .ok_or_else(|| {
7908            invalid_recorded_history(
7909                "timer_history_field_missing",
7910                sequence,
7911                field,
7912                &event.event_type,
7913                "timer history is missing a required numeric field",
7914            )
7915        })
7916}
7917
7918fn invalid_recorded_history(
7919    reason: &str,
7920    sequence: u64,
7921    expected: &str,
7922    actual: &str,
7923    message: &str,
7924) -> Error {
7925    Error::NonDeterministicReplay(ReplayFailure::new(
7926        reason,
7927        Some(sequence),
7928        Some(expected.to_string()),
7929        Some(actual.to_string()),
7930        message,
7931    ))
7932}
7933
7934type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7935
7936fn activity_outcome(
7937    event: &HistoryEvent,
7938    fallback_codec: &str,
7939    recorded_activity_type: Option<String>,
7940) -> Result<ActivityOutcome> {
7941    if event.event_type == "ActivityCompleted" {
7942        let codec = event
7943            .payload
7944            .get("payload_codec")
7945            .and_then(Value::as_str)
7946            .unwrap_or(fallback_codec);
7947        return Ok(Ok(decode_wire_avro_value(
7948            event.payload.get("result").unwrap_or(&Value::Null),
7949            codec,
7950        )?));
7951    }
7952
7953    let payload = &event.payload;
7954    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7955        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7956        "ActivityCancelled" => (
7957            ActivityFailureKind::Cancelled,
7958            "cancelled",
7959            "activity was cancelled",
7960        ),
7961        "ActivityTimedOut" => (
7962            ActivityFailureKind::TimedOut,
7963            "timeout",
7964            "activity timed out",
7965        ),
7966        _ => unreachable!("activity_outcome is called only for terminal activity events"),
7967    };
7968    let exception = payload
7969        .get("exception")
7970        .filter(|value| !value.is_null())
7971        .cloned();
7972    let failure_category = payload_string(payload, "failure_category");
7973    let timeout_kind = payload_string(payload, "timeout_kind");
7974    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7975        ActivityFailureKind::Failed => failure_category
7976            .clone()
7977            .unwrap_or_else(|| fallback_reason.to_string()),
7978        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7979        ActivityFailureKind::TimedOut => timeout_kind
7980            .clone()
7981            .unwrap_or_else(|| fallback_reason.to_string()),
7982    });
7983    let message = payload_string(payload, "message")
7984        .or_else(|| {
7985            exception
7986                .as_ref()
7987                .and_then(|value| payload_string(value, "message"))
7988        })
7989        .unwrap_or_else(|| fallback_message.to_string());
7990
7991    Ok(Err(ActivityFailure {
7992        kind,
7993        reason,
7994        message,
7995        activity_execution_id: payload_string(payload, "activity_execution_id"),
7996        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7997        activity_type: payload_string(payload, "activity_type")
7998            .or_else(|| payload_string(payload, "activity_name"))
7999            .or(recorded_activity_type),
8000        activity_class: payload_string(payload, "activity_class"),
8001        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
8002        failure_id: payload_string(payload, "failure_id"),
8003        failure_category,
8004        timeout_kind,
8005        non_retryable: payload
8006            .get("non_retryable")
8007            .and_then(Value::as_bool)
8008            .unwrap_or(false),
8009        exception_type: payload_string(payload, "exception_type").or_else(|| {
8010            exception
8011                .as_ref()
8012                .and_then(|value| payload_string(value, "type"))
8013        }),
8014        exception_class: payload_string(payload, "exception_class").or_else(|| {
8015            exception
8016                .as_ref()
8017                .and_then(|value| payload_string(value, "class"))
8018        }),
8019        code: payload
8020            .get("code")
8021            .filter(|value| !value.is_null())
8022            .cloned(),
8023        exception,
8024    }))
8025}
8026
8027type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
8028
8029fn child_workflow_outcomes(
8030    events: &[HistoryEvent],
8031    fallback_codec: &str,
8032    parent: WorkflowIdentity,
8033) -> Result<Vec<ChildWorkflowOutcome>> {
8034    let mut outcomes = Vec::new();
8035
8036    for event in events {
8037        let kind = match event.event_type.as_str() {
8038            "ChildRunCompleted" => None,
8039            "ChildRunFailed" => Some((
8040                ChildWorkflowFailureKind::Failed,
8041                "child_workflow",
8042                "child workflow failed",
8043            )),
8044            "ChildRunCancelled" => Some((
8045                ChildWorkflowFailureKind::Cancelled,
8046                "cancelled",
8047                "child workflow was cancelled",
8048            )),
8049            "ChildRunTerminated" => Some((
8050                ChildWorkflowFailureKind::Terminated,
8051                "terminated",
8052                "child workflow was terminated",
8053            )),
8054            _ => continue,
8055        };
8056        let payload = &event.payload;
8057        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
8058        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
8059        let child_workflow_type = payload_string(payload, "child_workflow_type");
8060
8061        if let Some((kind, reason, fallback_message)) = kind {
8062            let exception = payload
8063                .get("exception")
8064                .filter(|value| !value.is_null())
8065                .cloned();
8066            let message = payload_string(payload, "message")
8067                .or_else(|| {
8068                    exception
8069                        .as_ref()
8070                        .and_then(|value| payload_string(value, "message"))
8071                })
8072                .unwrap_or_else(|| fallback_message.to_string());
8073            let exception_type = payload_string(payload, "exception_type").or_else(|| {
8074                exception
8075                    .as_ref()
8076                    .and_then(|value| payload_string(value, "type"))
8077            });
8078            let exception_class = payload_string(payload, "exception_class").or_else(|| {
8079                exception
8080                    .as_ref()
8081                    .and_then(|value| payload_string(value, "class"))
8082            });
8083            outcomes.push(Err(ChildWorkflowFailure {
8084                kind,
8085                reason: reason.to_string(),
8086                message,
8087                parent_workflow_id: parent.workflow_id.clone(),
8088                parent_workflow_run_id: parent.run_id.clone(),
8089                child_workflow_id,
8090                child_workflow_run_id,
8091                child_workflow_type,
8092                failure_id: payload_string(payload, "failure_id"),
8093                failure_category: payload_string(payload, "failure_category"),
8094                exception_type,
8095                exception_class,
8096                non_retryable: payload
8097                    .get("non_retryable")
8098                    .and_then(Value::as_bool)
8099                    .unwrap_or(false),
8100                code: payload
8101                    .get("code")
8102                    .filter(|value| !value.is_null())
8103                    .cloned(),
8104                exception,
8105            }));
8106            continue;
8107        }
8108
8109        let codec = payload
8110            .get("payload_codec")
8111            .and_then(Value::as_str)
8112            .unwrap_or(fallback_codec);
8113        let result = payload
8114            .get("result")
8115            .or_else(|| payload.get("output"))
8116            .unwrap_or(&Value::Null);
8117        outcomes.push(Ok(ChildWorkflowAvroResult {
8118            parent: parent.clone(),
8119            child: WorkflowIdentity {
8120                workflow_id: child_workflow_id,
8121                run_id: child_workflow_run_id,
8122            },
8123            child_workflow_type,
8124            result: decode_wire_avro_value(result, codec)?,
8125        }));
8126    }
8127
8128    Ok(outcomes)
8129}
8130
8131fn payload_string(payload: &Value, key: &str) -> Option<String> {
8132    payload
8133        .get(key)
8134        .and_then(Value::as_str)
8135        .filter(|value| !value.is_empty())
8136        .map(str::to_string)
8137}
8138
8139fn workflow_failure_command(error: &Error) -> Value {
8140    let (exception_type, exception_class, properties) = match error {
8141        Error::ActivityFailed(failure) => (
8142            match failure.kind {
8143                ActivityFailureKind::Failed => "ActivityFailed",
8144                ActivityFailureKind::Cancelled => "ActivityCancelled",
8145                ActivityFailureKind::TimedOut => "ActivityTimedOut",
8146            },
8147            "durable_workflow::ActivityFailure",
8148            json!({
8149                "reason": failure.reason,
8150                "activity_execution_id": failure.activity_execution_id,
8151                "activity_attempt_id": failure.activity_attempt_id,
8152                "activity_type": failure.activity_type,
8153                "activity_class": failure.activity_class,
8154                "attempt_number": failure.attempt_number,
8155                "failure_id": failure.failure_id,
8156                "failure_category": failure.failure_category,
8157                "timeout_kind": failure.timeout_kind,
8158                "activity_non_retryable": failure.non_retryable,
8159                "activity_exception_type": failure.exception_type,
8160                "activity_exception_class": failure.exception_class,
8161                "activity_code": failure.code,
8162                "activity_exception": failure.exception,
8163            }),
8164        ),
8165        Error::ChildWorkflowFailed(failure) => (
8166            match failure.kind {
8167                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
8168                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
8169                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
8170            },
8171            "durable_workflow::ChildWorkflowFailure",
8172            json!({
8173                "reason": failure.reason,
8174                "parent_workflow_id": failure.parent_workflow_id,
8175                "parent_workflow_run_id": failure.parent_workflow_run_id,
8176                "child_workflow_id": failure.child_workflow_id,
8177                "child_workflow_run_id": failure.child_workflow_run_id,
8178                "child_workflow_type": failure.child_workflow_type,
8179                "failure_id": failure.failure_id,
8180                "failure_category": failure.failure_category,
8181                "child_exception_type": failure.exception_type,
8182                "child_exception_class": failure.exception_class,
8183                "child_non_retryable": failure.non_retryable,
8184                "child_code": failure.code,
8185                "child_exception": failure.exception,
8186            }),
8187        ),
8188        Error::NonDeterministicReplay(_) => (
8189            "NonDeterministicReplay",
8190            "durable_workflow::Error",
8191            Value::Null,
8192        ),
8193        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
8194    };
8195    let non_retryable = match error {
8196        Error::ActivityFailed(failure) => failure.non_retryable,
8197        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
8198        Error::NonDeterministicReplay(_) => true,
8199        _ => false,
8200    };
8201
8202    json!({
8203        "type": "fail_workflow",
8204        "message": error.to_string(),
8205        "exception_type": exception_type,
8206        "exception_class": exception_class,
8207        "non_retryable": non_retryable,
8208        "exception": {
8209            "type": exception_type,
8210            "class": exception_class,
8211            "message": error.to_string(),
8212            "properties": properties,
8213        }
8214    })
8215}
8216
8217fn workflow_task_integrity_error(error: &Error) -> bool {
8218    matches!(
8219        error,
8220        Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
8221    )
8222}
8223
8224fn decode_signal_event_arguments(
8225    event: &HistoryEvent,
8226    fallback_codec: &str,
8227) -> Result<Vec<AvroValue>> {
8228    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
8229    validate_payload_codec(codec)?;
8230    let raw = signal_history_payload(&event.payload);
8231    let decoded = match raw.filter(|value| !value.is_null()) {
8232        Some(value) => decode_wire_avro_value(value, codec)?,
8233        None => AvroValue::Array(Vec::new()),
8234    };
8235    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
8236        unreachable!("normalize_avro_arguments always returns an array");
8237    };
8238    Ok(arguments)
8239}
8240
8241fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8242    let Some(export_events) = task
8243        .history_export
8244        .as_ref()
8245        .and_then(|export| export.get("history_events"))
8246        .and_then(Value::as_array)
8247    else {
8248        return Ok(());
8249    };
8250
8251    if export_events.len() > task.history_events.len() {
8252        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8253    }
8254
8255    Ok(())
8256}
8257
8258fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8259    let Some(export) = task.history_export.as_ref() else {
8260        return Ok(());
8261    };
8262    let signals = export
8263        .get("signals")
8264        .and_then(Value::as_array)
8265        .cloned()
8266        .unwrap_or_default();
8267    let activities = export
8268        .get("activities")
8269        .and_then(Value::as_array)
8270        .cloned()
8271        .unwrap_or_default();
8272    let export_codec = export
8273        .get("payloads")
8274        .and_then(|payloads| payloads.get("codec"))
8275        .and_then(Value::as_str)
8276        .unwrap_or(&task.payload_codec)
8277        .to_string();
8278    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8279
8280    for event in &mut task.history_events {
8281        if event.event_type == "ActivityCompleted" {
8282            let sequence = event
8283                .payload
8284                .get("sequence")
8285                .or_else(|| event.payload.get("workflow_sequence"))
8286                .and_then(value_as_u64);
8287            let Some(activity) = sequence.and_then(|sequence| {
8288                activities.iter().find(|activity| {
8289                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8290                })
8291            }) else {
8292                continue;
8293            };
8294            let Some(payload) = event.payload.as_object_mut() else {
8295                continue;
8296            };
8297            if missing_payload(payload.get("result")) {
8298                if let Some(result) = activity
8299                    .get("result")
8300                    .filter(|value| !missing_payload(Some(value)))
8301                {
8302                    payload.insert("result".to_string(), result.clone());
8303                }
8304            }
8305            for field in ["payload_codec", "activity_type"] {
8306                if payload
8307                    .get(field)
8308                    .and_then(Value::as_str)
8309                    .unwrap_or_default()
8310                    .is_empty()
8311                {
8312                    if let Some(value) = activity.get(field) {
8313                        payload.insert(field.to_string(), value.clone());
8314                    }
8315                }
8316            }
8317            continue;
8318        }
8319
8320        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8321            continue;
8322        }
8323        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8324        let command_id = event
8325            .payload
8326            .get("workflow_command_id")
8327            .or_else(|| event.raw.get("workflow_command_id"))
8328            .and_then(Value::as_str);
8329        let signal_name = event
8330            .payload
8331            .get("signal_name")
8332            .and_then(Value::as_str)
8333            .unwrap_or_default()
8334            .to_string();
8335        let matched = signals
8336            .iter()
8337            .find(|signal| {
8338                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8339            })
8340            .or_else(|| {
8341                signals.iter().find(|signal| {
8342                    command_id.is_some()
8343                        && signal.get("command_id").and_then(Value::as_str) == command_id
8344                })
8345            })
8346            .or_else(|| {
8347                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8348                let signal = signals
8349                    .iter()
8350                    .filter(|signal| {
8351                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8352                    })
8353                    .nth(*offset);
8354                if signal.is_some() {
8355                    *offset += 1;
8356                }
8357                signal
8358            });
8359        let Some(signal) = matched else {
8360            continue;
8361        };
8362        let signal_codec = signal
8363            .get("payload_codec")
8364            .and_then(Value::as_str)
8365            .unwrap_or(&export_codec);
8366        let Some(payload) = event.payload.as_object_mut() else {
8367            continue;
8368        };
8369        if missing_payload(payload.get("arguments")) {
8370            if let Some(arguments) = signal
8371                .get("arguments")
8372                .filter(|value| !missing_payload(Some(value)))
8373            {
8374                let envelope = match arguments {
8375                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8376                    other => other.clone(),
8377                };
8378                payload.insert("arguments".to_string(), envelope);
8379            }
8380        }
8381        if payload
8382            .get("payload_codec")
8383            .and_then(Value::as_str)
8384            .unwrap_or_default()
8385            .is_empty()
8386        {
8387            payload.insert("payload_codec".to_string(), json!(signal_codec));
8388        }
8389    }
8390
8391    Ok(())
8392}
8393
8394fn missing_payload(value: Option<&Value>) -> bool {
8395    match value {
8396        None | Some(Value::Null) => true,
8397        Some(Value::String(value)) => value.is_empty(),
8398        Some(_) => false,
8399    }
8400}
8401
8402fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8403    let export_signals = task
8404        .history_export
8405        .as_ref()
8406        .and_then(|export| export.get("signals"))
8407        .and_then(Value::as_array)
8408        .cloned()
8409        .unwrap_or_default();
8410    let export_codec = task
8411        .history_export
8412        .as_ref()
8413        .and_then(|export| export.get("payloads"))
8414        .and_then(|payloads| payloads.get("codec"))
8415        .and_then(Value::as_str)
8416        .unwrap_or(&task.payload_codec);
8417    let mut name_offsets: HashMap<String, usize> = HashMap::new();
8418    let mut signals = Vec::new();
8419
8420    for event in &task.history_events {
8421        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8422            continue;
8423        }
8424
8425        let name = event
8426            .payload
8427            .get("signal_name")
8428            .and_then(Value::as_str)
8429            .unwrap_or_default();
8430        if name.is_empty() {
8431            continue;
8432        }
8433        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8434        let command_id = event
8435            .payload
8436            .get("workflow_command_id")
8437            .or_else(|| event.raw.get("workflow_command_id"))
8438            .and_then(Value::as_str);
8439        let matched_export = export_signals
8440            .iter()
8441            .find(|candidate| {
8442                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8443            })
8444            .or_else(|| {
8445                export_signals.iter().find(|candidate| {
8446                    command_id.is_some()
8447                        && candidate.get("command_id").and_then(Value::as_str) == command_id
8448                })
8449            })
8450            .or_else(|| {
8451                let offset = name_offsets.entry(name.to_string()).or_default();
8452                let candidate = export_signals
8453                    .iter()
8454                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8455                    .nth(*offset);
8456                if candidate.is_some() {
8457                    *offset += 1;
8458                }
8459                candidate
8460            });
8461        let codec = event
8462            .payload
8463            .get("payload_codec")
8464            .and_then(Value::as_str)
8465            .or_else(|| {
8466                matched_export
8467                    .and_then(|signal| signal.get("payload_codec"))
8468                    .and_then(Value::as_str)
8469            })
8470            .unwrap_or(export_codec);
8471        let raw_arguments = signal_history_payload(&event.payload)
8472            .filter(|value| !value.is_null())
8473            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8474        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8475        let workflow_sequence = event
8476            .payload
8477            .get("workflow_sequence")
8478            .and_then(value_as_u64)
8479            .or_else(|| {
8480                matched_export
8481                    .and_then(|signal| signal.get("workflow_sequence"))
8482                    .and_then(value_as_u64)
8483            });
8484
8485        signals.push(QuerySignal {
8486            id: signal_id.map(str::to_string).or_else(|| {
8487                matched_export
8488                    .and_then(|signal| signal.get("id"))
8489                    .and_then(Value::as_str)
8490                    .map(str::to_string)
8491            }),
8492            name: name.to_string(),
8493            arguments,
8494            avro_arguments,
8495            workflow_sequence,
8496        });
8497    }
8498
8499    if signals.is_empty() {
8500        for signal in export_signals {
8501            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8502                continue;
8503            }
8504            let Some(name) = signal.get("name").and_then(Value::as_str) else {
8505                continue;
8506            };
8507            let codec = signal
8508                .get("payload_codec")
8509                .and_then(Value::as_str)
8510                .unwrap_or(export_codec);
8511            let (arguments, avro_arguments) =
8512                decode_query_signal_arguments(signal.get("arguments"), codec)?;
8513            signals.push(QuerySignal {
8514                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8515                name: name.to_string(),
8516                arguments,
8517                avro_arguments,
8518                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8519            });
8520        }
8521        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8522    }
8523
8524    Ok(signals)
8525}
8526
8527fn decode_query_signal_arguments(
8528    raw: Option<&Value>,
8529    codec: &str,
8530) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8531    validate_payload_codec(codec)?;
8532    let decoded = match raw.filter(|value| !value.is_null()) {
8533        Some(value) => decode_wire_avro_value(value, codec)?,
8534        None => AvroValue::Array(Vec::new()),
8535    };
8536    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8537        unreachable!("normalize_avro_arguments always returns an array");
8538    };
8539    let arguments = avro_arguments
8540        .iter()
8541        .cloned()
8542        .map(AvroValue::into_json)
8543        .collect::<Result<Vec<_>>>()?;
8544    Ok((arguments, avro_arguments))
8545}
8546
8547fn value_as_u64(value: &Value) -> Option<u64> {
8548    value
8549        .as_u64()
8550        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8551}
8552
8553#[cfg(test)]
8554mod tests {
8555    use super::*;
8556    use std::{
8557        io::{Read, Write},
8558        net::{SocketAddr, TcpListener, TcpStream},
8559        sync::atomic::AtomicUsize,
8560        thread,
8561    };
8562
8563    #[derive(Clone, Copy, Debug)]
8564    enum InvalidTaskPayloadCodec {
8565        Missing,
8566        Null,
8567        NonString,
8568    }
8569
8570    impl InvalidTaskPayloadCodec {
8571        fn label(self) -> &'static str {
8572            match self {
8573                Self::Missing => "missing",
8574                Self::Null => "null",
8575                Self::NonString => "non-string",
8576            }
8577        }
8578
8579        fn apply(self, task: &mut Value) {
8580            let task = task.as_object_mut().expect("task fixture object");
8581            match self {
8582                Self::Missing => {
8583                    task.remove("payload_codec");
8584                }
8585                Self::Null => {
8586                    task.insert("payload_codec".to_string(), Value::Null);
8587                }
8588                Self::NonString => {
8589                    task.insert("payload_codec".to_string(), json!(42));
8590                }
8591            }
8592        }
8593    }
8594
8595    fn fixture_envelope(value: Value) -> Value {
8596        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
8597    }
8598
8599    fn fixture_blob(value: Value) -> String {
8600        encode_payload(&value, DEFAULT_CODEC)
8601            .expect("encode Avro test fixture")
8602            .blob
8603    }
8604
8605    #[test]
8606    fn client_builder_rejects_the_sdk_owned_api_suffix() {
8607        for base_url in [
8608            "http://127.0.0.1:8080/api",
8609            "http://localhost:8080/api/",
8610            "https://runtime.example.test/namespaces/orders/api",
8611        ] {
8612            let error = Client::builder(base_url)
8613                .build()
8614                .expect_err("SDK-owned /api suffix must be rejected during build");
8615
8616            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8617            assert!(
8618                error.to_string().contains("SDK appends /api automatically"),
8619                "the validation error must explain how to fix the endpoint"
8620            );
8621        }
8622    }
8623
8624    #[test]
8625    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8626        for (base_url, expected) in [
8627            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8628            (
8629                "http://localhost:8080/durable-workflow/",
8630                "http://localhost:8080/durable-workflow",
8631            ),
8632            (
8633                "https://runtime.example.test/namespaces/orders",
8634                "https://runtime.example.test/namespaces/orders",
8635            ),
8636            (
8637                "https://runtime.example.test/gateway/api/namespaces/orders",
8638                "https://runtime.example.test/gateway/api/namespaces/orders",
8639            ),
8640            (
8641                "https://api.example.test/runtime/orders/",
8642                "https://api.example.test/runtime/orders",
8643            ),
8644        ] {
8645            let client = Client::builder(base_url)
8646                .build()
8647                .expect("Server and Cloud runtime base URL must remain valid");
8648
8649            assert_eq!(client.base_url, expected);
8650        }
8651    }
8652
8653    fn typed_fidelity_probe() -> AvroValue {
8654        AvroValue::Map(BTreeMap::from([
8655            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8656            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8657            (
8658                "numeric".to_string(),
8659                AvroValue::Map(BTreeMap::from([
8660                    ("0".to_string(), AvroValue::String("zero".to_string())),
8661                    ("1".to_string(), AvroValue::String("one".to_string())),
8662                ])),
8663            ),
8664            (
8665                "nested".to_string(),
8666                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8667                    "enabled".to_string(),
8668                    AvroValue::Boolean(true),
8669                )]))]),
8670            ),
8671            (
8672                "projection_collisions".to_string(),
8673                AvroValue::Array(projection_collision_probe()),
8674            ),
8675        ]))
8676    }
8677
8678    fn projection_collision_probe() -> Vec<AvroValue> {
8679        vec![
8680            AvroValue::Map(BTreeMap::from([
8681                ("$type".to_string(), AvroValue::String("bytes".to_string())),
8682                (
8683                    "base64".to_string(),
8684                    AvroValue::String("ordinary user text".to_string()),
8685                ),
8686            ])),
8687            AvroValue::Map(BTreeMap::from([
8688                ("$type".to_string(), AvroValue::String("map".to_string())),
8689                (
8690                    "entries".to_string(),
8691                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8692                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
8693                        (
8694                            "value".to_string(),
8695                            AvroValue::String("user map".to_string()),
8696                        ),
8697                    ]))]),
8698                ),
8699            ])),
8700        ]
8701    }
8702
8703    #[derive(Clone, Debug, Default, PartialEq)]
8704    struct ReplayCounterState {
8705        loaded: Option<String>,
8706        count: i64,
8707        finished: bool,
8708    }
8709
8710    fn replay_counter_worker() -> Worker {
8711        let client = Client::new("http://127.0.0.1:8080").expect("client");
8712        let mut worker = Worker::new(client, "rust-workers");
8713        worker.register_replayed_workflow(
8714            "replay-counter",
8715            ReplayCounterState::default,
8716            |ctx, _input, state| async move {
8717                let loaded = ctx.activity("load-counter", json!([])).await?;
8718                state.update(|current| {
8719                    current.loaded = loaded.as_str().map(str::to_string);
8720                })?;
8721                for _ in 0..2 {
8722                    let signal = ctx.wait_signal("increment").await?;
8723                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8724                    state.update(|current| current.count += amount)?;
8725                }
8726                state.update(|current| current.finished = true)?;
8727                state.read(|current| Ok(json!(current.count)))?
8728            },
8729        );
8730        worker.register_replayed_query::<ReplayCounterState, _, _>(
8731            "replay-counter",
8732            "current",
8733            |_ctx, state, _args| async move {
8734                Ok(json!({
8735                    "loaded": state.loaded,
8736                    "count": state.count,
8737                    "finished": state.finished,
8738                }))
8739            },
8740        );
8741        worker.register_replayed_query::<ReplayCounterState, _, _>(
8742            "replay-counter",
8743            "detached-mutation",
8744            |_ctx, state, _args| async move {
8745                let mut detached = (*state).clone();
8746                detached.count = 999;
8747                Ok(json!(detached.count))
8748            },
8749        );
8750        worker.register_replayed_query::<ReplayCounterState, _, _>(
8751            "replay-counter",
8752            "failed-mutation",
8753            |_ctx, state, _args| async move {
8754                let mut detached = (*state).clone();
8755                detached.count = 999;
8756                Err(Error::WorkerLoop("query refused".to_string()))
8757            },
8758        );
8759        worker
8760    }
8761
8762    fn replay_counter_query(
8763        query_name: &str,
8764        history_events: Value,
8765        run_status: &str,
8766    ) -> QueryTask {
8767        let arguments = fixture_envelope(json!([]));
8768        serde_json::from_value(json!({
8769            "query_task_id": format!("query-{query_name}"),
8770            "workflow_type": "replay-counter",
8771            "query_name": query_name,
8772            "payload_codec": DEFAULT_CODEC,
8773            "workflow_arguments": arguments.clone(),
8774            "query_arguments": arguments,
8775            "history_events": history_events,
8776            "run_status": run_status,
8777        }))
8778        .expect("query task")
8779    }
8780
8781    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8782        workflow_context_with_codec(history, DEFAULT_CODEC)
8783    }
8784
8785    fn workflow_context_with_codec(
8786        history: Vec<HistoryEvent>,
8787        payload_codec: &str,
8788    ) -> WorkflowContext {
8789        WorkflowContext {
8790            state: Arc::new(Mutex::new(
8791                WorkflowState::new_with_identity(
8792                    history,
8793                    None,
8794                    None,
8795                    "rust-workers".to_string(),
8796                    payload_codec.to_string(),
8797                    None,
8798                )
8799                .expect("valid workflow history"),
8800            )),
8801        }
8802    }
8803
8804    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8805        HistoryEvent {
8806            event_type: event_type.to_string(),
8807            payload,
8808            raw: HashMap::new(),
8809        }
8810    }
8811
8812    fn workflow_task(
8813        workflow_type: &str,
8814        history_events: Vec<HistoryEvent>,
8815        payload_codec: &str,
8816    ) -> WorkflowTask {
8817        WorkflowTask {
8818            task_id: format!("wft-{workflow_type}"),
8819            workflow_id: Some(format!("wf-{workflow_type}")),
8820            run_id: Some(format!("run-{workflow_type}")),
8821            workflow_type: workflow_type.to_string(),
8822            payload_codec: payload_codec.to_string(),
8823            arguments: Some(
8824                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8825            ),
8826            total_history_events: Some(history_events.len() as u64),
8827            history_size_bytes: None,
8828            continue_as_new_recommended: None,
8829            history_budget_pressure: None,
8830            history_events,
8831            next_history_page_token: None,
8832            workflow_task_attempt: 1,
8833            workflow_signal_id: None,
8834            signal_name: None,
8835            signal_arguments: None,
8836            workflow_update_id: None,
8837            update_name: None,
8838            lease_owner: Some("rust-worker".to_string()),
8839        }
8840    }
8841
8842    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8843    struct SideEffectProbe {
8844        request_id: String,
8845        attempt: u32,
8846    }
8847
8848    #[test]
8849    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8850        let calls = AtomicUsize::new(0);
8851        let ctx = workflow_context(Vec::new());
8852        let value = ctx
8853            .side_effect(|| {
8854                calls.fetch_add(1, Ordering::SeqCst);
8855                SideEffectProbe {
8856                    request_id: "request-42".to_string(),
8857                    attempt: 3,
8858                }
8859            })
8860            .expect("first side effect");
8861        assert_eq!(value.attempt, 3);
8862        assert_eq!(calls.load(Ordering::SeqCst), 1);
8863        let commands = ctx.take_commands().expect("commands");
8864        assert_eq!(commands.len(), 1);
8865        assert_eq!(commands[0]["type"], "record_side_effect");
8866        assert_eq!(
8867            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8868            serde_json::to_value(&value).expect("value")
8869        );
8870
8871        let replay = workflow_context(vec![history_event(
8872            "SideEffectRecorded",
8873            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8874        )]);
8875        let replayed: SideEffectProbe = replay
8876            .side_effect(|| {
8877                calls.fetch_add(1, Ordering::SeqCst);
8878                panic!("committed side-effect callbacks must not run during replay")
8879            })
8880            .expect("replayed side effect");
8881        assert_eq!(replayed, value);
8882        assert_eq!(calls.load(Ordering::SeqCst), 1);
8883        assert!(replay.take_commands().expect("commands").is_empty());
8884        replay.ensure_history_consumed().expect("history consumed");
8885    }
8886
8887    #[test]
8888    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8889        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8890        let value = ctx
8891            .side_effect(|| SideEffectProbe {
8892                request_id: "avro-request".to_string(),
8893                attempt: 1,
8894            })
8895            .expect("Avro side effect");
8896        let uuid = ctx.uuid_v4().expect("deterministic UUID");
8897        let commands = ctx.take_commands().expect("commands");
8898        assert_eq!(commands.len(), 2);
8899        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8900        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8901        assert_eq!(
8902            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8903            serde_json::to_value(&value).expect("value")
8904        );
8905
8906        let replay = workflow_context_with_codec(
8907            vec![
8908                history_event(
8909                    "SideEffectRecorded",
8910                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8911                ),
8912                history_event(
8913                    "SideEffectRecorded",
8914                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8915                ),
8916            ],
8917            DEFAULT_CODEC,
8918        );
8919        let replayed: SideEffectProbe = replay
8920            .side_effect(|| panic!("Avro callback must not run"))
8921            .expect("replayed Avro value");
8922        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8923        assert_eq!(replayed, value);
8924        assert_eq!(replayed_uuid, uuid);
8925        assert!(replay.take_commands().expect("commands").is_empty());
8926    }
8927
8928    #[test]
8929    fn typed_side_effect_replay_preserves_bytes_and_maps() {
8930        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8931        let value = ctx
8932            .side_effect_avro_value(typed_fidelity_probe)
8933            .expect("typed side effect");
8934        let commands = ctx.take_commands().expect("side-effect command");
8935        assert_eq!(
8936            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8937                .expect("recorded side effect"),
8938            value
8939        );
8940
8941        let replay = workflow_context_with_codec(
8942            vec![history_event(
8943                "SideEffectRecorded",
8944                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8945            )],
8946            DEFAULT_CODEC,
8947        );
8948        assert_eq!(
8949            replay
8950                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8951                .expect("replayed typed side effect"),
8952            value
8953        );
8954    }
8955
8956    #[test]
8957    fn ordered_side_effects_share_the_durable_command_stream() {
8958        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
8959        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
8960        let ctx = workflow_context(vec![
8961            history_event(
8962                "SideEffectRecorded",
8963                json!({"sequence": 1, "result": first}),
8964            ),
8965            history_event(
8966                "SideEffectRecorded",
8967                json!({"sequence": 2, "result": second}),
8968            ),
8969        ]);
8970        let first: String = ctx
8971            .side_effect(|| panic!("first callback must not run"))
8972            .expect("first replay");
8973        let second: i32 = ctx
8974            .side_effect(|| panic!("second callback must not run"))
8975            .expect("second replay");
8976        assert_eq!(first, "first");
8977        assert_eq!(second, 29);
8978        ctx.ensure_history_consumed().expect("ordered history");
8979
8980        let reordered = workflow_context(vec![history_event(
8981            "VersionMarkerRecorded",
8982            json!({
8983                "sequence": 1,
8984                "change_id": "before-side-effect",
8985                "version": 1,
8986                "min_supported": 1,
8987                "max_supported": 1,
8988            }),
8989        )]);
8990        let error = reordered
8991            .side_effect(|| "new".to_string())
8992            .expect_err("command reordering must fail");
8993        assert!(matches!(
8994            error,
8995            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8996                if reason == "recorded_command_mismatch"
8997        ));
8998    }
8999
9000    #[test]
9001    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
9002        let ctx = workflow_context(Vec::new());
9003        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
9004        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
9005        assert!(ctx.patched("new-search").expect("patch"));
9006        ctx.deprecate_patch("new-search").expect("deprecate patch");
9007        let commands = ctx.take_commands().expect("commands");
9008        assert_eq!(commands.len(), 2);
9009        assert_eq!(commands[0]["type"], "record_version_marker");
9010        assert_eq!(commands[0]["version"], 2);
9011        assert_eq!(commands[1]["change_id"], "new-search");
9012
9013        let replay = workflow_context(vec![history_event(
9014            "VersionMarkerRecorded",
9015            json!({
9016                "sequence": 1,
9017                "change_id": "checkout-v2",
9018                "version": 2,
9019                "min_supported": 1,
9020                "max_supported": 2,
9021            }),
9022        )]);
9023        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
9024        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
9025        assert!(replay.take_commands().expect("commands").is_empty());
9026        replay.ensure_history_consumed().expect("history consumed");
9027    }
9028
9029    #[test]
9030    fn version_markers_reject_incompatible_or_malformed_history() {
9031        let incompatible = workflow_context(vec![history_event(
9032            "VersionMarkerRecorded",
9033            json!({
9034                "sequence": 1,
9035                "change_id": "checkout-v2",
9036                "version": 1,
9037                "min_supported": 1,
9038                "max_supported": 2,
9039            }),
9040        )]);
9041        let error = incompatible
9042            .get_version("checkout-v2", 2, 3)
9043            .expect_err("old version is unsupported");
9044        assert!(matches!(
9045            error,
9046            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9047                if reason == "version_marker_incompatible_range"
9048        ));
9049
9050        for (history, reason) in [
9051            (
9052                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
9053                "side_effect_result_missing",
9054            ),
9055            (
9056                vec![history_event(
9057                    "SideEffectRecorded",
9058                    json!({
9059                        "sequence": 1,
9060                        "result": {"codec": "avro", "blob": "not-base64"},
9061                    }),
9062                )],
9063                "side_effect_payload_incompatible",
9064            ),
9065            (
9066                vec![history_event(
9067                    "SideEffectRecorded",
9068                    json!({"sequence": 1, "result": {"unwrapped": true}}),
9069                )],
9070                "side_effect_payload_malformed",
9071            ),
9072            (
9073                vec![history_event(
9074                    "VersionMarkerRecorded",
9075                    json!({
9076                        "sequence": 1,
9077                        "change_id": "change",
9078                        "version": 1,
9079                        "min_supported": 2,
9080                        "max_supported": 1,
9081                    }),
9082                )],
9083                "version_marker_history_range_invalid",
9084            ),
9085        ] {
9086            let error = WorkflowState::new(
9087                history,
9088                "rust-workers".to_string(),
9089                DEFAULT_CODEC.to_string(),
9090                None,
9091            )
9092            .expect_err("malformed history must fail");
9093            assert!(matches!(
9094                error,
9095                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
9096                    if actual == reason
9097            ));
9098        }
9099    }
9100
9101    #[test]
9102    fn duplicate_side_effects_and_version_markers_are_rejected() {
9103        let duplicate_side_effect = WorkflowState::new(
9104            vec![
9105                history_event(
9106                    "SideEffectRecorded",
9107                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
9108                ),
9109                history_event(
9110                    "SideEffectRecorded",
9111                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
9112                ),
9113            ],
9114            "rust-workers".to_string(),
9115            DEFAULT_CODEC.to_string(),
9116            None,
9117        )
9118        .expect_err("duplicate side effect");
9119        assert!(matches!(
9120            duplicate_side_effect,
9121            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9122                if reason == "duplicate_side_effect_record"
9123        ));
9124
9125        let marker = |sequence| {
9126            history_event(
9127                "VersionMarkerRecorded",
9128                json!({
9129                    "sequence": sequence,
9130                    "change_id": "same-change",
9131                    "version": 1,
9132                    "min_supported": 1,
9133                    "max_supported": 1,
9134                }),
9135            )
9136        };
9137        let duplicate_marker = WorkflowState::new(
9138            vec![marker(1), marker(3)],
9139            "rust-workers".to_string(),
9140            DEFAULT_CODEC.to_string(),
9141            None,
9142        )
9143        .expect_err("duplicate marker");
9144        assert!(matches!(
9145            duplicate_marker,
9146            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9147                if reason == "duplicate_version_marker"
9148        ));
9149    }
9150
9151    #[test]
9152    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
9153        fn worker(calls: Arc<AtomicUsize>) -> Worker {
9154            let client = Client::new("http://127.0.0.1:8080").expect("client");
9155            let mut worker = Worker::new(client, "rust-workers");
9156            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
9157                let calls = Arc::clone(&calls);
9158                async move {
9159                    let captured = ctx.side_effect(|| {
9160                        calls.fetch_add(1, Ordering::SeqCst);
9161                        "captured-once".to_string()
9162                    })?;
9163                    let version = ctx.get_version("cold-restart", 1, 2)?;
9164                    Ok(json!({"captured": captured, "version": version}))
9165                }
9166            });
9167            worker
9168        }
9169
9170        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
9171            WorkflowTask {
9172                task_id: "wft-side-effect-version".to_string(),
9173                workflow_id: Some("wf-side-effect-version".to_string()),
9174                run_id: Some("run-side-effect-version".to_string()),
9175                workflow_type: "rust.side-effect-version".to_string(),
9176                payload_codec: DEFAULT_CODEC.to_string(),
9177                arguments: Some(
9178                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
9179                ),
9180                history_events,
9181                total_history_events: None,
9182                history_size_bytes: None,
9183                continue_as_new_recommended: None,
9184                history_budget_pressure: None,
9185                next_history_page_token: None,
9186                workflow_task_attempt: 1,
9187                workflow_signal_id: None,
9188                signal_name: None,
9189                signal_arguments: None,
9190                workflow_update_id: None,
9191                update_name: None,
9192                lease_owner: Some("rust-worker".to_string()),
9193            }
9194        }
9195
9196        let calls = Arc::new(AtomicUsize::new(0));
9197        let initial = worker(Arc::clone(&calls))
9198            .execute_workflow_task(task(Vec::new()))
9199            .expect("initial execution");
9200        assert_eq!(
9201            initial
9202                .iter()
9203                .map(|command| &command["type"])
9204                .collect::<Vec<_>>(),
9205            vec![
9206                "record_side_effect",
9207                "record_version_marker",
9208                "complete_workflow"
9209            ]
9210        );
9211        assert_eq!(calls.load(Ordering::SeqCst), 1);
9212
9213        let restarted = worker(Arc::clone(&calls));
9214        let replayed = restarted
9215            .execute_workflow_task(task(vec![
9216                history_event(
9217                    "SideEffectRecorded",
9218                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
9219                ),
9220                history_event(
9221                    "VersionMarkerRecorded",
9222                    json!({
9223                        "sequence": 2,
9224                        "change_id": "cold-restart",
9225                        "version": 2,
9226                        "min_supported": 1,
9227                        "max_supported": 2,
9228                    }),
9229                ),
9230            ]))
9231            .expect("cold replay");
9232        assert_eq!(replayed.len(), 1);
9233        assert_eq!(replayed[0]["type"], "complete_workflow");
9234        assert_eq!(calls.load(Ordering::SeqCst), 1);
9235    }
9236
9237    #[test]
9238    fn side_effect_replay_rejects_changed_rust_value_type() {
9239        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
9240        let ctx = workflow_context(vec![history_event(
9241            "SideEffectRecorded",
9242            json!({"sequence": 1, "result": result}),
9243        )]);
9244        let error = ctx
9245            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
9246            .expect_err("changed type must fail replay");
9247        assert!(matches!(
9248            error,
9249            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
9250                if reason == "side_effect_type_mismatch"
9251        ));
9252    }
9253
9254    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
9255        vec![
9256            history_event(
9257                "ActivityScheduled",
9258                json!({
9259                    "sequence": 1,
9260                    "activity_type": "flaky",
9261                    "activity_execution_id": "act-1",
9262                    "activity": {
9263                        "id": "act-1",
9264                        "sequence": 1,
9265                        "type": "flaky",
9266                        "queue": "critical-activities",
9267                        "execution_mode": null,
9268                        "retry_policy": {
9269                            "snapshot_version": 1,
9270                            "max_attempts": 3,
9271                            "backoff_seconds": [2, 4],
9272                            "start_to_close_timeout": 30,
9273                            "schedule_to_start_timeout": 5,
9274                            "schedule_to_close_timeout": 90,
9275                            "heartbeat_timeout": 10,
9276                            "non_retryable_error_types": ["PermanentError"]
9277                        }
9278                    }
9279                }),
9280            ),
9281            history_event(
9282                "ActivityStarted",
9283                json!({
9284                    "sequence": 1,
9285                    "activity_type": "flaky",
9286                    "activity_execution_id": "act-1",
9287                    "activity_attempt_id": "attempt-1",
9288                    "attempt_number": 1
9289                }),
9290            ),
9291            history_event(
9292                "ActivityRetryScheduled",
9293                json!({
9294                    "sequence": 1,
9295                    "activity_type": "flaky",
9296                    "activity_execution_id": "act-1",
9297                    "activity_attempt_id": "attempt-1",
9298                    "attempt_number": 1,
9299                    "retry_after_attempt": 1,
9300                    "retry_backoff_seconds": 2,
9301                    "failure_category": "activity",
9302                    "exception_type": "TransientError"
9303                }),
9304            ),
9305            history_event(
9306                "ActivityStarted",
9307                json!({
9308                    "sequence": 1,
9309                    "activity_type": "flaky",
9310                    "activity_execution_id": "act-1",
9311                    "activity_attempt_id": "attempt-2",
9312                    "attempt_number": 2
9313                }),
9314            ),
9315            history_event(
9316                "ActivityCompleted",
9317                json!({
9318                    "sequence": 1,
9319                    "activity_type": "flaky",
9320                    "activity_execution_id": "act-1",
9321                    "activity_attempt_id": "attempt-2",
9322                    "attempt_number": 2,
9323                    "payload_codec": DEFAULT_CODEC,
9324                    "result": fixture_envelope(json!({"status":"recovered"}))
9325                }),
9326            ),
9327        ]
9328    }
9329
9330    fn retry_activity_options() -> ActivityOptions {
9331        ActivityOptions::new()
9332            .task_queue("critical-activities")
9333            .retry_policy(
9334                ActivityRetryPolicy::new(3)
9335                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9336                    .non_retryable_error_type("PermanentError"),
9337            )
9338            .start_to_close_timeout(Duration::from_secs(30))
9339            .schedule_to_start_timeout(Duration::from_secs(5))
9340            .schedule_to_close_timeout(Duration::from_secs(90))
9341            .heartbeat_timeout(Duration::from_secs(10))
9342    }
9343
9344    #[test]
9345    fn fixed_avro_value_round_trips_json_values() {
9346        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9347        let envelope = PayloadEnvelope::avro(&value).expect("encode");
9348        assert_eq!(envelope.codec, DEFAULT_CODEC);
9349        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9350    }
9351
9352    #[tokio::test]
9353    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9354        let client = Client::new("http://127.0.0.1:8080").expect("client");
9355        let mut worker = Worker::new(client, "rust-workers");
9356        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9357        worker
9358            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9359        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9360            Ok(input)
9361        });
9362        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9363            Ok(input)
9364        });
9365        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9366            Ok(AvroValue::Array(
9367                ctx.wait_signal_avro_value("changed").await?,
9368            ))
9369        });
9370
9371        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9372        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9373
9374        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9375        workflow.arguments = Some(envelope.clone());
9376        let commands = worker
9377            .execute_workflow_task(workflow)
9378            .expect("typed workflow task");
9379        assert_eq!(commands[0]["type"], "complete_workflow");
9380        assert_eq!(
9381            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9382                .expect("typed workflow result"),
9383            arguments
9384        );
9385
9386        let activity = ActivityTask {
9387            task_id: "activity-typed".to_string(),
9388            activity_attempt_id: Some("attempt-typed".to_string()),
9389            attempt_id: None,
9390            activity_type: "typed.activity".to_string(),
9391            payload_codec: DEFAULT_CODEC.to_string(),
9392            arguments: Some(envelope.clone()),
9393            attempt_number: 1,
9394            lease_owner: Some("rust-worker".to_string()),
9395        };
9396        assert_eq!(
9397            worker
9398                .execute_activity_task(activity)
9399                .await
9400                .expect("typed activity result"),
9401            arguments
9402        );
9403
9404        let query = QueryTask {
9405            query_task_id: "query-typed".to_string(),
9406            query_task_attempt: 1,
9407            lease_owner: Some("rust-worker".to_string()),
9408            workflow_id: Some("typed-1".to_string()),
9409            run_id: Some("run-typed".to_string()),
9410            workflow_type: "typed.echo".to_string(),
9411            query_name: "inspect".to_string(),
9412            payload_codec: DEFAULT_CODEC.to_string(),
9413            workflow_arguments: Some(
9414                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9415                    .expect("workflow input"),
9416            ),
9417            query_arguments: Some(envelope.clone()),
9418            history_events: Vec::new(),
9419            history_export: None,
9420            run_status: Some("running".to_string()),
9421        };
9422        assert_eq!(
9423            worker
9424                .execute_query_task(query)
9425                .await
9426                .expect("typed query result"),
9427            arguments
9428        );
9429
9430        let mut update = workflow_task(
9431            "typed.echo",
9432            vec![history_event(
9433                "UpdateAccepted",
9434                json!({
9435                    "update_id": "update-typed",
9436                    "update_name": "replace",
9437                    "arguments": envelope.clone(),
9438                }),
9439            )],
9440            DEFAULT_CODEC,
9441        );
9442        update.workflow_update_id = Some("update-typed".to_string());
9443        update.update_name = Some("replace".to_string());
9444        let commands = worker
9445            .execute_workflow_task(update)
9446            .expect("typed update task");
9447        assert_eq!(commands[0]["type"], "complete_update");
9448        assert_eq!(
9449            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9450                .expect("typed update result"),
9451            arguments
9452        );
9453
9454        let mut signal = workflow_task(
9455            "typed.signal",
9456            vec![history_event(
9457                "SignalReceived",
9458                json!({
9459                    "signal_id": "signal-typed",
9460                    "signal_name": "changed",
9461                    "arguments": envelope.clone(),
9462                }),
9463            )],
9464            DEFAULT_CODEC,
9465        );
9466        signal.workflow_signal_id = Some("signal-typed".to_string());
9467        signal.signal_name = Some("changed".to_string());
9468        signal.signal_arguments = Some(envelope);
9469        let commands = worker
9470            .execute_workflow_task(signal)
9471            .expect("typed signal resume");
9472        assert_eq!(
9473            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9474                .expect("typed signal result"),
9475            arguments
9476        );
9477    }
9478
9479    #[tokio::test]
9480    async fn typed_helpers_never_parse_json_inspection_projection() {
9481        let collision_values = projection_collision_probe();
9482        let expected = AvroValue::Array(collision_values.clone());
9483        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9484
9485        let activity_context = workflow_context_with_codec(
9486            vec![history_event(
9487                "ActivityCompleted",
9488                json!({
9489                    "sequence": 1,
9490                    "activity_type": "collision.activity",
9491                    "payload_codec": DEFAULT_CODEC,
9492                    "result": envelope.clone(),
9493                }),
9494            )],
9495            DEFAULT_CODEC,
9496        );
9497        assert_eq!(
9498            activity_context
9499                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9500                .await
9501                .expect("typed activity collision result"),
9502            expected
9503        );
9504
9505        let signal_context = workflow_context_with_codec(
9506            vec![
9507                history_event(
9508                    "SignalWaitOpened",
9509                    json!({"sequence": 1, "signal_name": "collision"}),
9510                ),
9511                history_event(
9512                    "SignalApplied",
9513                    json!({
9514                        "sequence": 1,
9515                        "signal_name": "collision",
9516                        "payload_codec": DEFAULT_CODEC,
9517                        "value": envelope.clone(),
9518                    }),
9519                ),
9520            ],
9521            DEFAULT_CODEC,
9522        );
9523        assert_eq!(
9524            signal_context
9525                .wait_signal_avro_value("collision")
9526                .await
9527                .expect("typed signal collision arguments"),
9528            collision_values
9529        );
9530
9531        let child_context = workflow_context_with_codec(
9532            vec![
9533                history_event(
9534                    "ChildWorkflowScheduled",
9535                    json!({
9536                        "sequence": 1,
9537                        "child_workflow_instance_id": "collision-child",
9538                        "child_workflow_run_id": "collision-run",
9539                        "child_workflow_type": "collision.child",
9540                    }),
9541                ),
9542                history_event(
9543                    "ChildRunCompleted",
9544                    json!({
9545                        "sequence": 1,
9546                        "child_workflow_instance_id": "collision-child",
9547                        "child_workflow_run_id": "collision-run",
9548                        "child_workflow_type": "collision.child",
9549                        "payload_codec": DEFAULT_CODEC,
9550                        "result": envelope,
9551                    }),
9552                ),
9553            ],
9554            DEFAULT_CODEC,
9555        );
9556        let child = child_context
9557            .start_child_workflow_avro_value(
9558                "collision.child",
9559                ChildWorkflowOptions::new("collision-workers"),
9560                AvroValue::Array(Vec::new()),
9561            )
9562            .await
9563            .expect("typed child collision result");
9564        assert_eq!(child.result, expected);
9565    }
9566
9567    #[tokio::test]
9568    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9569        let client = Client::new("http://127.0.0.1:8080").expect("client");
9570        let mut worker = Worker::new(client, "rust-workers");
9571        worker.register_replayed_workflow_avro_value(
9572            "typed.replayed",
9573            || (),
9574            |_ctx, input, _state| async move { Ok(input) },
9575        );
9576        worker.register_replayed_query_avro_value::<(), _, _>(
9577            "typed.replayed",
9578            "inspect",
9579            |ctx, _state, args| async move {
9580                let mut signals = ctx.signals_avro_value("collision");
9581                let signal = signals
9582                    .pop()
9583                    .map(AvroValue::Array)
9584                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9585                Ok(AvroValue::Array(vec![
9586                    ctx.workflow_input_avro_value().clone(),
9587                    signal,
9588                    args,
9589                ]))
9590            },
9591        );
9592        let arguments = AvroValue::Array(projection_collision_probe());
9593        let signal_arguments =
9594            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9595        let task = QueryTask {
9596            query_task_id: "query-typed-replay".to_string(),
9597            query_task_attempt: 1,
9598            lease_owner: Some("rust-worker".to_string()),
9599            workflow_id: Some("typed-replay".to_string()),
9600            run_id: Some("run-typed-replay".to_string()),
9601            workflow_type: "typed.replayed".to_string(),
9602            query_name: "inspect".to_string(),
9603            payload_codec: DEFAULT_CODEC.to_string(),
9604            workflow_arguments: Some(
9605                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9606            ),
9607            query_arguments: Some(
9608                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9609            ),
9610            history_events: vec![history_event(
9611                "SignalReceived",
9612                json!({
9613                    "signal_id": "collision-signal",
9614                    "signal_name": "collision",
9615                    "workflow_sequence": 1,
9616                    "payload_codec": DEFAULT_CODEC,
9617                    "arguments": signal_arguments,
9618                }),
9619            )],
9620            history_export: None,
9621            run_status: Some("completed".to_string()),
9622        };
9623
9624        assert_eq!(
9625            worker
9626                .execute_query_task(task)
9627                .await
9628                .expect("typed replay query"),
9629            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9630        );
9631    }
9632
9633    #[test]
9634    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9635        let value = BTreeMap::from([(1_i32, "integer key")]);
9636        let error = PayloadEnvelope::avro(&value)
9637            .expect_err("integer map keys must fail")
9638            .to_string();
9639
9640        assert!(error.contains("invalid_map_key"));
9641    }
9642
9643    #[test]
9644    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
9645        let envelope = PayloadEnvelope {
9646            codec: "json".to_string(),
9647            blob: r#"{"greeting":"hello"}"#.to_string(),
9648        };
9649
9650        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
9651        let diagnostic = error.to_string();
9652        assert!(diagnostic.contains("unsupported_payload_codec"));
9653        assert!(diagnostic.contains("codec=\"avro\""));
9654        assert!(diagnostic.contains("HTTP document transport"));
9655    }
9656
9657    #[test]
9658    fn untagged_json_payload_value_fails_closed() {
9659        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
9660            .expect_err("untagged JSON payload values must fail");
9661        let diagnostic = error.to_string();
9662        assert!(diagnostic.contains("unsupported_payload_codec"));
9663        assert!(diagnostic.contains("untagged durable payload"));
9664        assert!(diagnostic.contains("HTTP document transport"));
9665    }
9666
9667    #[test]
9668    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9669        let envelope = PayloadEnvelope {
9670            codec: DEFAULT_CODEC.to_string(),
9671            blob: BASE64.encode([0x01]),
9672        };
9673
9674        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9675        assert!(error.to_string().contains("invalid_payload_framing"));
9676    }
9677
9678    #[tokio::test]
9679    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
9680        let server = MockWorkerServer::start();
9681        let client = Client::builder(server.base_url())
9682            .timeout(Duration::from_secs(2))
9683            .build()
9684            .expect("client");
9685        let invalid_commands = [
9686            json!({
9687                "type": "complete_workflow",
9688                "result": {"codec": "json", "blob": null}
9689            }),
9690            json!({
9691                "type": "schedule_activity",
9692                "arguments": {"codec": "yaml", "blob": "ignored"}
9693            }),
9694            json!({
9695                "type": "start_child_workflow",
9696                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
9697            }),
9698            json!({"type": "continue_as_new", "arguments": []}),
9699            json!({"type": "complete_update"}),
9700            json!({"type": "record_side_effect", "result": null}),
9701            json!({
9702                "type": "start_service_operation",
9703                "payload_codec": DEFAULT_CODEC,
9704                "request_payload": "raw-avro-bytes"
9705            }),
9706        ];
9707
9708        for command in invalid_commands {
9709            let error = client
9710                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
9711                .await
9712                .expect_err("invalid durable payload must fail locally");
9713            let diagnostic = error.to_string();
9714            assert!(
9715                diagnostic.contains("unsupported_payload_codec")
9716                    || diagnostic.contains("invalid_payload_envelope")
9717                    || diagnostic.contains("untagged durable payload"),
9718                "unexpected validation diagnostic: {diagnostic}"
9719            );
9720        }
9721
9722        assert_eq!(
9723            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
9724            0,
9725            "invalid command payloads must not reach HTTP transport"
9726        );
9727    }
9728
9729    #[test]
9730    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
9731        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
9732        let commands = [
9733            json!({"type": "complete_workflow", "result": envelope.clone()}),
9734            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
9735            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
9736            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
9737            json!({"type": "complete_update", "result": envelope.clone()}),
9738            json!({"type": "record_side_effect", "result": envelope.clone()}),
9739            json!({
9740                "type": "start_service_operation",
9741                "payload_codec": DEFAULT_CODEC,
9742                "request_payload": envelope.clone()
9743            }),
9744            json!({
9745                "type": "complete_workflow",
9746                "result": envelope,
9747                "metadata": {
9748                    "codec": "json",
9749                    "payload_codec": "customer-codec",
9750                    "result": {"codec": "yaml", "blob": null}
9751                }
9752            }),
9753        ];
9754
9755        validate_workflow_task_commands(&commands)
9756            .expect("customer metadata must not become a protocol codec declaration");
9757    }
9758
9759    #[test]
9760    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
9761        assert_eq!(
9762            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
9763            AvroValue::Array(Vec::new())
9764        );
9765        assert_eq!(
9766            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
9767            AvroValue::Array(Vec::new())
9768        );
9769
9770        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
9771        signal.signal_name = Some("empty-signal".to_string());
9772        signal.signal_arguments = None;
9773        let decoded = decode_resume_signal(&signal)
9774            .expect("valid Avro signal")
9775            .expect("named signal resumes the workflow");
9776        assert!(decoded.arguments.is_empty());
9777    }
9778
9779    #[tokio::test]
9780    async fn malformed_task_level_codecs_become_pre_handler_failures() {
9781        let client = Client::new("http://127.0.0.1:8080").expect("client");
9782        let mut worker = Worker::new(client, "rust-workers");
9783        let handler_calls = Arc::new(AtomicUsize::new(0));
9784
9785        let calls = Arc::clone(&handler_calls);
9786        worker.register_workflow("codec.workflow", move |_ctx, _args| {
9787            calls.fetch_add(1, Ordering::SeqCst);
9788            async move { Ok(Value::Null) }
9789        });
9790        let calls = Arc::clone(&handler_calls);
9791        worker.register_activity("codec.activity", move |_ctx, _args| {
9792            calls.fetch_add(1, Ordering::SeqCst);
9793            async move { Ok(Value::Null) }
9794        });
9795        let calls = Arc::clone(&handler_calls);
9796        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
9797            calls.fetch_add(1, Ordering::SeqCst);
9798            async move { Ok(Value::Null) }
9799        });
9800
9801        let mut failures = Vec::new();
9802        for codec_case in [
9803            InvalidTaskPayloadCodec::Missing,
9804            InvalidTaskPayloadCodec::Null,
9805            InvalidTaskPayloadCodec::NonString,
9806        ] {
9807            let mut workflow = json!({
9808                "task_id": format!("workflow-{}", codec_case.label()),
9809                "workflow_type": "codec.workflow"
9810            });
9811            codec_case.apply(&mut workflow);
9812            match serde_json::from_value::<WorkflowTask>(workflow) {
9813                Ok(task) => match worker.execute_workflow_task(task) {
9814                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
9815                    outcome => failures.push(format!(
9816                        "workflow {} codec returned {outcome:?}",
9817                        codec_case.label()
9818                    )),
9819                },
9820                Err(error) => failures.push(format!(
9821                    "workflow {} codec failed transport deserialization: {error}",
9822                    codec_case.label()
9823                )),
9824            }
9825
9826            let mut activity = json!({
9827                "task_id": format!("activity-{}", codec_case.label()),
9828                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
9829                "activity_type": "codec.activity",
9830                "attempt_number": 1
9831            });
9832            codec_case.apply(&mut activity);
9833            match serde_json::from_value::<ActivityTask>(activity) {
9834                Ok(task) => match worker.execute_activity_task(task).await {
9835                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
9836                    outcome => failures.push(format!(
9837                        "activity {} codec returned {outcome:?}",
9838                        codec_case.label()
9839                    )),
9840                },
9841                Err(error) => failures.push(format!(
9842                    "activity {} codec failed transport deserialization: {error}",
9843                    codec_case.label()
9844                )),
9845            }
9846
9847            let mut query = json!({
9848                "query_task_id": format!("query-{}", codec_case.label()),
9849                "workflow_type": "codec.workflow",
9850                "query_name": "known"
9851            });
9852            codec_case.apply(&mut query);
9853            match serde_json::from_value::<QueryTask>(query) {
9854                Ok(task) => match worker.execute_query_task(task).await {
9855                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
9856                    outcome => failures.push(format!(
9857                        "query {} codec returned {outcome:?}",
9858                        codec_case.label()
9859                    )),
9860                },
9861                Err(error) => failures.push(format!(
9862                    "query {} codec failed transport deserialization: {error}",
9863                    codec_case.label()
9864                )),
9865            }
9866        }
9867
9868        assert!(failures.is_empty(), "{}", failures.join("\n"));
9869        assert_eq!(
9870            handler_calls.load(Ordering::SeqCst),
9871            0,
9872            "invalid task codecs must not invoke a handler"
9873        );
9874    }
9875
9876    #[tokio::test]
9877    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
9878        for codec_case in [
9879            InvalidTaskPayloadCodec::Missing,
9880            InvalidTaskPayloadCodec::Null,
9881            InvalidTaskPayloadCodec::NonString,
9882        ] {
9883            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
9884            let client = Client::builder(server.base_url())
9885                .timeout(Duration::from_secs(2))
9886                .build()
9887                .expect("client");
9888            let mut worker = Worker::new(client, "rust-workers")
9889                .worker_id("codec-worker")
9890                .poll_timeout(Duration::from_millis(10));
9891            let handler_calls = Arc::new(AtomicUsize::new(0));
9892
9893            let calls = Arc::clone(&handler_calls);
9894            worker.register_workflow("codec.workflow", move |_ctx, _args| {
9895                calls.fetch_add(1, Ordering::SeqCst);
9896                async move { Ok(Value::Null) }
9897            });
9898            let calls = Arc::clone(&handler_calls);
9899            worker.register_activity("codec.activity", move |_ctx, _args| {
9900                calls.fetch_add(1, Ordering::SeqCst);
9901                async move { Ok(Value::Null) }
9902            });
9903            let calls = Arc::clone(&handler_calls);
9904            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
9905                calls.fetch_add(1, Ordering::SeqCst);
9906                async move { Ok(Value::Null) }
9907            });
9908
9909            assert_eq!(
9910                worker.run_once().await.expect("invalid tasks are settled"),
9911                3,
9912                "all {} codec tasks must be handled",
9913                codec_case.label()
9914            );
9915            assert_eq!(
9916                handler_calls.load(Ordering::SeqCst),
9917                0,
9918                "{} task codecs must fail before every handler",
9919                codec_case.label()
9920            );
9921
9922            for path in [
9923                "/api/worker/workflow-tasks/codec-workflow/fail",
9924                "/api/worker/activity-tasks/codec-activity/fail",
9925                "/api/worker/query-tasks/codec-query/fail",
9926            ] {
9927                let body = server.request_body(path);
9928                assert!(
9929                    body["failure"]["message"]
9930                        .as_str()
9931                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
9932                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
9933                    codec_case.label()
9934                );
9935            }
9936            assert_eq!(
9937                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
9938                    ["reason"],
9939                "query_payload_decode_failed"
9940            );
9941            for path in [
9942                "/api/worker/workflow-tasks/codec-workflow/complete",
9943                "/api/worker/activity-tasks/codec-activity/complete",
9944                "/api/worker/query-tasks/codec-query/complete",
9945            ] {
9946                assert_eq!(
9947                    server.request_count(path),
9948                    0,
9949                    "invalid {} codec task reached {path}",
9950                    codec_case.label()
9951                );
9952            }
9953        }
9954    }
9955
9956    #[tokio::test]
9957    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
9958        let client = Client::new("http://127.0.0.1:8080").expect("client");
9959        let mut worker = Worker::new(client, "rust-workers");
9960        let handler_calls = Arc::new(AtomicUsize::new(0));
9961
9962        let calls = Arc::clone(&handler_calls);
9963        worker.register_workflow("codec.workflow", move |_ctx, _args| {
9964            calls.fetch_add(1, Ordering::SeqCst);
9965            async move { Ok(Value::Null) }
9966        });
9967        let calls = Arc::clone(&handler_calls);
9968        worker.register_activity("codec.activity", move |_ctx, _args| {
9969            calls.fetch_add(1, Ordering::SeqCst);
9970            async move { Ok(Value::Null) }
9971        });
9972        let calls = Arc::clone(&handler_calls);
9973        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
9974            calls.fetch_add(1, Ordering::SeqCst);
9975            async move { Ok(Value::Null) }
9976        });
9977        let calls = Arc::clone(&handler_calls);
9978        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
9979            calls.fetch_add(1, Ordering::SeqCst);
9980            async move { Ok(Value::Null) }
9981        });
9982
9983        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
9984        workflow.payload_codec = "json".to_string();
9985        workflow.arguments = None;
9986        let error = worker
9987            .execute_workflow_task(workflow)
9988            .expect_err("task codec must be checked before workflow invocation");
9989        assert!(error.to_string().contains("unsupported_payload_codec"));
9990
9991        let activity = ActivityTask {
9992            task_id: "activity-invalid-codec".to_string(),
9993            activity_attempt_id: None,
9994            attempt_id: None,
9995            activity_type: "codec.activity".to_string(),
9996            payload_codec: "unknown".to_string(),
9997            arguments: None,
9998            attempt_number: 1,
9999            lease_owner: None,
10000        };
10001        let error = worker
10002            .execute_activity_task(activity)
10003            .await
10004            .expect_err("task codec must be checked before activity invocation");
10005        assert!(error.to_string().contains("unsupported_payload_codec"));
10006
10007        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
10008        update.workflow_update_id = Some("update-invalid-codec".to_string());
10009        update.update_name = Some("known".to_string());
10010        update.history_events.push(history_event(
10011            "UpdateAccepted",
10012            json!({
10013                "update_id": "update-invalid-codec",
10014                "update_name": "known",
10015                "arguments": {"codec": "json", "blob": null}
10016            }),
10017        ));
10018        let error = worker
10019            .execute_workflow_task(update)
10020            .expect_err("nested update codec must be checked before handler lookup");
10021        assert!(error.to_string().contains("unsupported_payload_codec"));
10022
10023        let query: QueryTask = serde_json::from_value(json!({
10024            "query_task_id": "query-invalid-codec",
10025            "workflow_type": "codec.workflow",
10026            "query_name": "known",
10027            "payload_codec": DEFAULT_CODEC,
10028            "workflow_arguments": null,
10029            "query_arguments": null,
10030            "history_export": {
10031                "payloads": {"codec": DEFAULT_CODEC},
10032                "signals": [{
10033                    "name": "empty",
10034                    "payload_codec": "json",
10035                    "arguments": null
10036                }]
10037            }
10038        }))
10039        .expect("query task");
10040        let failure = worker
10041            .execute_query_task(query)
10042            .await
10043            .expect_err("exported signal codec must be checked before query invocation");
10044        assert_eq!(failure.reason, "query_payload_decode_failed");
10045        assert!(failure.message.contains("unsupported_payload_codec"));
10046
10047        let exported_history: QueryTask = serde_json::from_value(json!({
10048            "query_task_id": "query-invalid-history-codec",
10049            "workflow_type": "codec.workflow",
10050            "query_name": "known",
10051            "payload_codec": DEFAULT_CODEC,
10052            "history_export": {
10053                "payloads": {"codec": DEFAULT_CODEC},
10054                "history_events": [{
10055                    "type": "ActivityCompleted",
10056                    "payload": {"payload_codec": "unknown", "result": null}
10057                }]
10058            }
10059        }))
10060        .expect("query task");
10061        let failure = worker
10062            .execute_query_task(exported_history)
10063            .await
10064            .expect_err("exported history codec must be checked before query invocation");
10065        assert_eq!(failure.reason, "query_payload_decode_failed");
10066        assert!(failure.message.contains("unsupported_payload_codec"));
10067        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
10068
10069        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
10070        unknown_workflow.arguments = None;
10071        unknown_workflow.history_events.push(history_event(
10072            "SignalReceived",
10073            json!({
10074                "signal_name": "empty",
10075                "payload_codec": "json",
10076                "arguments": null
10077            }),
10078        ));
10079        let error = worker
10080            .execute_workflow_task(unknown_workflow)
10081            .expect_err("history codec must precede unknown workflow outcome");
10082        assert!(error.to_string().contains("unsupported_payload_codec"));
10083
10084        let unknown_activity = ActivityTask {
10085            task_id: "activity-unknown".to_string(),
10086            activity_attempt_id: None,
10087            attempt_id: None,
10088            activity_type: "missing".to_string(),
10089            payload_codec: "json".to_string(),
10090            arguments: None,
10091            attempt_number: 1,
10092            lease_owner: None,
10093        };
10094        let error = worker
10095            .execute_activity_task(unknown_activity)
10096            .await
10097            .expect_err("codec must precede unknown activity outcome");
10098        assert!(error.to_string().contains("unsupported_payload_codec"));
10099
10100        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
10101        unknown_update.payload_codec = "json".to_string();
10102        unknown_update.arguments = None;
10103        unknown_update.workflow_update_id = Some("update-unknown".to_string());
10104        unknown_update.update_name = Some("missing".to_string());
10105        let error = worker
10106            .execute_workflow_task(unknown_update)
10107            .expect_err("codec must precede fail_update shortcut");
10108        assert!(error.to_string().contains("unsupported_payload_codec"));
10109
10110        let unknown_query: QueryTask = serde_json::from_value(json!({
10111            "query_task_id": "query-unknown",
10112            "workflow_type": "missing",
10113            "query_name": "missing",
10114            "payload_codec": "json",
10115            "workflow_arguments": null,
10116            "query_arguments": null
10117        }))
10118        .expect("query task");
10119        let failure = worker
10120            .execute_query_task(unknown_query)
10121            .await
10122            .expect_err("codec must precede unknown query outcome");
10123        assert_eq!(failure.reason, "query_payload_decode_failed");
10124        assert!(failure.message.contains("unsupported_payload_codec"));
10125    }
10126
10127    #[tokio::test]
10128    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
10129        let client = Client::new("http://127.0.0.1:8080").expect("client");
10130        let worker = Worker::new(client, "rust-workers");
10131
10132        for event_type in ["SignalReceived", "SignalApplied"] {
10133            for (payload_field, codec) in [
10134                ("value", "json"),
10135                ("input", "unknown"),
10136                ("arguments", "json"),
10137            ] {
10138                let payload = json!({
10139                    "signal_name": "empty",
10140                    payload_field: {"codec": codec, "blob": null}
10141                });
10142                let workflow = workflow_task(
10143                    "missing",
10144                    vec![history_event(event_type, payload.clone())],
10145                    DEFAULT_CODEC,
10146                );
10147                let error = worker
10148                    .execute_workflow_task(workflow)
10149                    .expect_err("signal payload codec must precede unknown workflow outcome");
10150                assert!(
10151                    error.to_string().contains("unsupported_payload_codec"),
10152                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
10153                );
10154
10155                let query: QueryTask = serde_json::from_value(json!({
10156                    "query_task_id": format!("query-{event_type}-{payload_field}"),
10157                    "workflow_type": "missing",
10158                    "query_name": "missing",
10159                    "payload_codec": DEFAULT_CODEC,
10160                    "workflow_arguments": null,
10161                    "query_arguments": null,
10162                    "history_events": [{
10163                        "event_type": event_type,
10164                        "payload": payload
10165                    }]
10166                }))
10167                .expect("query task");
10168                let failure = worker
10169                    .execute_query_task(query)
10170                    .await
10171                    .expect_err("signal payload codec must precede unknown query outcome");
10172                assert_eq!(
10173                    failure.reason, "query_payload_decode_failed",
10174                    "{event_type}.{payload_field} returned an unrelated query outcome"
10175                );
10176                assert!(
10177                    failure.message.contains("unsupported_payload_codec"),
10178                    "{event_type}.{payload_field} returned an unrelated query error: {}",
10179                    failure.message
10180                );
10181            }
10182        }
10183    }
10184
10185    #[test]
10186    fn workflow_context_schedules_activity_until_completion_is_in_history() {
10187        let ctx = WorkflowContext {
10188            state: Arc::new(Mutex::new(
10189                WorkflowState::new_with_identity(
10190                    Vec::new(),
10191                    Some("wf-parent".to_string()),
10192                    Some("run-parent".to_string()),
10193                    "rust-workers".to_string(),
10194                    DEFAULT_CODEC.to_string(),
10195                    None,
10196                )
10197                .expect("workflow state"),
10198            )),
10199        };
10200
10201        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
10202        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10203        assert!(matches!(
10204            call.as_mut().poll(&mut task_context),
10205            Poll::Pending
10206        ));
10207
10208        let commands = ctx.take_commands().expect("commands");
10209        assert_eq!(commands[0]["type"], "schedule_activity");
10210        assert_eq!(commands[0]["activity_type"], "hello.activity");
10211    }
10212
10213    #[test]
10214    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
10215        let ctx = workflow_context(Vec::new());
10216        let options = ActivityOptions::new()
10217            .task_queue("payments")
10218            .retry_policy(
10219                ActivityRetryPolicy::new(4)
10220                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
10221                    .non_retryable_error_type("ValidationError"),
10222            )
10223            .start_to_close_timeout(Duration::from_secs(120))
10224            .schedule_to_start_timeout(Duration::from_secs(10))
10225            .schedule_to_close_timeout(Duration::from_secs(300))
10226            .heartbeat_timeout(Duration::from_secs(15));
10227        let mut call = Box::pin(ctx.activity_with_options(
10228            "charge-card",
10229            options,
10230            json!([{"order_id": "o-1"}]),
10231        ));
10232        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10233
10234        assert!(matches!(
10235            call.as_mut().poll(&mut task_context),
10236            Poll::Pending
10237        ));
10238        assert!(matches!(
10239            call.as_mut().poll(&mut task_context),
10240            Poll::Pending
10241        ));
10242
10243        let commands = ctx.take_commands().expect("activity command");
10244        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
10245        assert_eq!(commands[0]["queue"], "payments");
10246        assert_eq!(
10247            commands[0]["retry_policy"],
10248            json!({
10249                "max_attempts": 4,
10250                "backoff_seconds": [1, 3, 9],
10251                "non_retryable_error_types": ["ValidationError"],
10252            })
10253        );
10254        assert_eq!(commands[0]["start_to_close_timeout"], 120);
10255        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
10256        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
10257        assert_eq!(commands[0]["heartbeat_timeout"], 15);
10258    }
10259
10260    #[test]
10261    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
10262        let ctx = workflow_context(Vec::new());
10263        let options = ActivityOptions::new().retry_policy(
10264            ActivityRetryPolicy::new(3)
10265                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
10266        );
10267        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
10268        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10269
10270        assert!(matches!(
10271            call.as_mut().poll(&mut task_context),
10272            Poll::Pending
10273        ));
10274        assert_eq!(
10275            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
10276            json!([1, 2])
10277        );
10278    }
10279
10280    #[test]
10281    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
10282        let cases = [
10283            (
10284                ActivityOptions::new().task_queue("  "),
10285                ActivityOptionsErrorKind::EmptyTaskQueue,
10286            ),
10287            (
10288                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
10289                ActivityOptionsErrorKind::EmptyRetryPolicy,
10290            ),
10291            (
10292                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
10293                ActivityOptionsErrorKind::InvalidMaxAttempts,
10294            ),
10295            (
10296                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
10297                    max_attempts: None,
10298                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
10299                    non_retryable_error_types: Vec::new(),
10300                }),
10301                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
10302            ),
10303            (
10304                ActivityOptions::new().retry_policy(
10305                    ActivityRetryPolicy::new(2)
10306                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
10307                ),
10308                ActivityOptionsErrorKind::TooManyBackoffIntervals,
10309            ),
10310            (
10311                ActivityOptions::new().retry_policy(
10312                    ActivityRetryPolicy::new(2).exponential_backoff(
10313                        Duration::from_secs(1),
10314                        0,
10315                        None,
10316                    ),
10317                ),
10318                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
10319            ),
10320            (
10321                ActivityOptions::new()
10322                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
10323                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
10324            ),
10325            (
10326                ActivityOptions::new().retry_policy(
10327                    ActivityRetryPolicy::new(10_002).exponential_backoff(
10328                        Duration::from_secs(1),
10329                        1,
10330                        None,
10331                    ),
10332                ),
10333                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
10334            ),
10335            (
10336                ActivityOptions::new().retry_policy(
10337                    ActivityRetryPolicy::new(2)
10338                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
10339                ),
10340                ActivityOptionsErrorKind::BackoffOverflow,
10341            ),
10342        ];
10343
10344        for (options, expected_kind) in cases {
10345            let ctx = workflow_context(Vec::new());
10346            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
10347            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10348            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
10349                call.as_mut().poll(&mut task_context)
10350            else {
10351                panic!("expected typed activity validation error");
10352            };
10353            assert_eq!(error.kind, expected_kind);
10354            assert!(ctx.take_commands().expect("commands").is_empty());
10355        }
10356    }
10357
10358    #[test]
10359    fn activity_options_validate_positive_and_ordered_timeouts() {
10360        let zero_timeout_cases = [
10361            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
10362            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
10363            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
10364            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
10365        ];
10366        for options in zero_timeout_cases {
10367            assert_eq!(
10368                options.validate().expect_err("zero timeout").kind,
10369                ActivityOptionsErrorKind::TimeoutNotPositive
10370            );
10371        }
10372
10373        let ordering_cases = [
10374            ActivityOptions::new()
10375                .heartbeat_timeout(Duration::from_secs(11))
10376                .start_to_close_timeout(Duration::from_secs(10)),
10377            ActivityOptions::new()
10378                .start_to_close_timeout(Duration::from_secs(31))
10379                .schedule_to_close_timeout(Duration::from_secs(30)),
10380            ActivityOptions::new()
10381                .schedule_to_start_timeout(Duration::from_secs(31))
10382                .schedule_to_close_timeout(Duration::from_secs(30)),
10383        ];
10384        for options in ordering_cases {
10385            assert_eq!(
10386                options.validate().expect_err("timeout order").kind,
10387                ActivityOptionsErrorKind::TimeoutOrder
10388            );
10389        }
10390
10391        assert_eq!(
10392            ActivityOptions::new()
10393                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
10394                .validate()
10395                .expect_err("protocol integer overflow")
10396                .kind,
10397            ActivityOptionsErrorKind::TimeoutOverflow
10398        );
10399    }
10400
10401    #[test]
10402    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
10403        let ctx = workflow_context(completed_retry_activity_history());
10404        let mut call =
10405            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10406        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10407
10408        assert!(matches!(
10409            call.as_mut().poll(&mut task_context),
10410            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10411        ));
10412        assert!(ctx.take_commands().expect("commands").is_empty());
10413        ctx.ensure_history_consumed().expect("history consumed");
10414    }
10415
10416    #[test]
10417    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
10418        let mut options = retry_activity_options();
10419        options
10420            .retry_policy
10421            .as_mut()
10422            .expect("retry policy")
10423            .non_retryable_error_types
10424            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
10425
10426        let new_ctx = workflow_context(Vec::new());
10427        let mut new_call =
10428            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
10429        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10430        assert!(matches!(
10431            new_call.as_mut().poll(&mut task_context),
10432            Poll::Pending
10433        ));
10434        let commands = new_ctx.take_commands().expect("commands");
10435        assert_eq!(commands.len(), 1);
10436        assert_eq!(
10437            commands[0]["retry_policy"]["non_retryable_error_types"],
10438            json!(["PermanentError"])
10439        );
10440
10441        let replay_ctx = workflow_context(completed_retry_activity_history());
10442        let mut replay_call =
10443            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
10444        assert!(matches!(
10445            replay_call.as_mut().poll(&mut task_context),
10446            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10447        ));
10448        assert!(replay_ctx.take_commands().expect("commands").is_empty());
10449        replay_ctx
10450            .ensure_history_consumed()
10451            .expect("history consumed");
10452    }
10453
10454    #[test]
10455    fn replayed_intermediate_retry_remains_pending_across_restarts() {
10456        let history = completed_retry_activity_history()
10457            .into_iter()
10458            .take(3)
10459            .collect::<Vec<_>>();
10460
10461        for _restart in 0..2 {
10462            let ctx = workflow_context(history.clone());
10463            let mut call =
10464                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10465            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10466            assert!(matches!(
10467                call.as_mut().poll(&mut task_context),
10468                Poll::Pending
10469            ));
10470            assert!(ctx.take_commands().expect("commands").is_empty());
10471        }
10472    }
10473
10474    #[test]
10475    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
10476        let mut changed_queue = retry_activity_options();
10477        changed_queue.task_queue = Some("different-queue".to_string());
10478
10479        let mut changed_max_attempts = retry_activity_options();
10480        let retry_policy = changed_max_attempts
10481            .retry_policy
10482            .as_mut()
10483            .expect("retry policy");
10484        retry_policy.max_attempts = Some(4);
10485
10486        let mut changed_backoff = retry_activity_options();
10487        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
10488        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
10489            Duration::from_secs(3),
10490            Duration::from_secs(4),
10491        ]));
10492
10493        let mut changed_non_retryable_types = retry_activity_options();
10494        let retry_policy = changed_non_retryable_types
10495            .retry_policy
10496            .as_mut()
10497            .expect("retry policy");
10498        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
10499
10500        let mut changed_start_to_close = retry_activity_options();
10501        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
10502        let mut changed_schedule_to_start = retry_activity_options();
10503        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
10504        let mut changed_schedule_to_close = retry_activity_options();
10505        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
10506        let mut changed_heartbeat = retry_activity_options();
10507        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
10508
10509        let cases = [
10510            (changed_queue, "activity_task_queue_mismatch"),
10511            (changed_max_attempts, "activity_retry_policy_mismatch"),
10512            (changed_backoff, "activity_retry_policy_mismatch"),
10513            (
10514                changed_non_retryable_types,
10515                "activity_retry_policy_mismatch",
10516            ),
10517            (changed_start_to_close, "activity_retry_policy_mismatch"),
10518            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
10519            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
10520            (changed_heartbeat, "activity_retry_policy_mismatch"),
10521        ];
10522
10523        for (options, expected_reason) in cases {
10524            let ctx = workflow_context(completed_retry_activity_history());
10525            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
10526            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10527            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10528                call.as_mut().poll(&mut task_context)
10529            else {
10530                panic!("changed activity options must fail replay");
10531            };
10532            assert_eq!(failure.reason, expected_reason);
10533            assert_eq!(failure.sequence, Some(1));
10534            assert!(ctx.take_commands().expect("commands").is_empty());
10535        }
10536    }
10537
10538    #[test]
10539    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
10540        let cases = [
10541            (
10542                "execution_mode",
10543                json!("local"),
10544                "activity_execution_mode_mismatch",
10545            ),
10546            (
10547                "snapshot_version",
10548                json!(2),
10549                "activity_retry_policy_mismatch",
10550            ),
10551        ];
10552
10553        for (field, value, expected_reason) in cases {
10554            let mut history = completed_retry_activity_history();
10555            let activity = history[0].payload["activity"]
10556                .as_object_mut()
10557                .expect("activity snapshot");
10558            if field == "execution_mode" {
10559                activity.insert(field.to_string(), value);
10560            } else {
10561                activity["retry_policy"]
10562                    .as_object_mut()
10563                    .expect("retry snapshot")
10564                    .insert(field.to_string(), value);
10565            }
10566
10567            let ctx = workflow_context(history);
10568            let mut call =
10569                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10570            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10571            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10572                call.as_mut().poll(&mut task_context)
10573            else {
10574                panic!("changed {field} must fail replay");
10575            };
10576            assert_eq!(failure.reason, expected_reason);
10577            assert_eq!(failure.sequence, Some(1));
10578            assert!(ctx.take_commands().expect("commands").is_empty());
10579        }
10580    }
10581
10582    #[test]
10583    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
10584        let mut history = completed_retry_activity_history();
10585        let activity = history[0].payload["activity"]
10586            .as_object_mut()
10587            .expect("activity snapshot");
10588        activity.remove("execution_mode");
10589        activity.remove("retry_policy");
10590
10591        let mut current = retry_activity_options();
10592        current.start_to_close_timeout = Some(Duration::from_secs(45));
10593        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
10594        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
10595        current.heartbeat_timeout = Some(Duration::from_secs(12));
10596
10597        let ctx = workflow_context(history);
10598        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
10599        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10600        assert!(matches!(
10601            call.as_mut().poll(&mut task_context),
10602            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
10603        ));
10604        assert!(ctx.take_commands().expect("commands").is_empty());
10605        ctx.ensure_history_consumed().expect("history consumed");
10606    }
10607
10608    #[test]
10609    fn terminal_activity_failed_after_start_returns_typed_failure() {
10610        let history = vec![
10611            history_event(
10612                "ActivityScheduled",
10613                json!({
10614                    "sequence": 1,
10615                    "activity_type": "flaky",
10616                    "activity_execution_id": "act-terminal",
10617                    "activity": {
10618                        "id": "act-terminal",
10619                        "sequence": 1,
10620                        "type": "flaky",
10621                        "queue": "critical-activities",
10622                        "retry_policy": {
10623                            "snapshot_version": 1,
10624                            "max_attempts": 3,
10625                            "backoff_seconds": [2, 4],
10626                            "non_retryable_error_types": ["PermanentError"]
10627                        }
10628                    }
10629                }),
10630            ),
10631            history_event(
10632                "ActivityStarted",
10633                json!({
10634                    "sequence": 1,
10635                    "activity_type": "flaky",
10636                    "activity_execution_id": "act-terminal",
10637                    "activity_attempt_id": "attempt-1",
10638                    "attempt_number": 1
10639                }),
10640            ),
10641            history_event(
10642                "ActivityFailed",
10643                json!({
10644                    "sequence": 1,
10645                    "activity_type": "flaky",
10646                    "activity_execution_id": "act-terminal",
10647                    "activity_attempt_id": "attempt-1",
10648                    "attempt_number": 1,
10649                    "failure_id": "failure-terminal",
10650                    "failure_category": "activity",
10651                    "exception_type": "PermanentError",
10652                    "message": "cannot retry",
10653                    "non_retryable": true
10654                }),
10655            ),
10656        ];
10657        let ctx = workflow_context(history);
10658        let mut call =
10659            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
10660        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10661
10662        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10663            call.as_mut().poll(&mut task_context)
10664        else {
10665            panic!("terminal ActivityFailed must settle the activity future");
10666        };
10667        assert_eq!(failure.kind, ActivityFailureKind::Failed);
10668        assert_eq!(
10669            failure.activity_execution_id.as_deref(),
10670            Some("act-terminal")
10671        );
10672        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
10673        assert!(failure.non_retryable);
10674        assert!(ctx.take_commands().expect("commands").is_empty());
10675        ctx.ensure_history_consumed().expect("history consumed");
10676    }
10677
10678    #[test]
10679    fn activity_terminal_events_return_machine_readable_failures() {
10680        let cases = [
10681            (
10682                "ActivityFailed",
10683                json!({
10684                    "sequence": 1,
10685                    "activity_type": "charge-card",
10686                    "activity_execution_id": "act-1",
10687                    "activity_attempt_id": "attempt-2",
10688                    "attempt_number": 2,
10689                    "failure_id": "failure-1",
10690                    "failure_category": "activity",
10691                    "exception_type": "PaymentDeclined",
10692                    "exception_class": "payments.PaymentDeclined",
10693                    "message": "card declined",
10694                    "non_retryable": true
10695                }),
10696                ActivityFailureKind::Failed,
10697                "activity",
10698            ),
10699            (
10700                "ActivityCancelled",
10701                json!({
10702                    "sequence": 1,
10703                    "activity_type": "charge-card",
10704                    "activity_execution_id": "act-1",
10705                    "activity_attempt_id": "attempt-1"
10706                }),
10707                ActivityFailureKind::Cancelled,
10708                "cancelled",
10709            ),
10710        ];
10711
10712        for (event_type, payload, expected_kind, expected_reason) in cases {
10713            let ctx = workflow_context(vec![history_event(event_type, payload)]);
10714            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
10715            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10716            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10717                call.as_mut().poll(&mut task_context)
10718            else {
10719                panic!("expected terminal activity failure");
10720            };
10721            assert_eq!(failure.kind, expected_kind);
10722            assert_eq!(failure.reason, expected_reason);
10723            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
10724            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
10725        }
10726    }
10727
10728    #[test]
10729    fn every_activity_timeout_class_is_typed() {
10730        for timeout_kind in [
10731            "start_to_close",
10732            "schedule_to_start",
10733            "schedule_to_close",
10734            "heartbeat",
10735        ] {
10736            let ctx = workflow_context(vec![history_event(
10737                "ActivityTimedOut",
10738                json!({
10739                    "sequence": 1,
10740                    "activity_type": "slow",
10741                    "activity_execution_id": "act-timeout",
10742                    "activity_attempt_id": "attempt-timeout",
10743                    "failure_category": "timeout",
10744                    "timeout_kind": timeout_kind,
10745                    "message": "deadline expired"
10746                }),
10747            )]);
10748            let mut call = Box::pin(ctx.activity("slow", json!([])));
10749            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10750            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
10751                call.as_mut().poll(&mut task_context)
10752            else {
10753                panic!("expected timeout failure");
10754            };
10755            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
10756            assert_eq!(failure.reason, timeout_kind);
10757            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
10758            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
10759        }
10760    }
10761
10762    #[test]
10763    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
10764        let ctx = workflow_context(Vec::new());
10765        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
10766        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10767
10768        assert!(matches!(
10769            sleep.as_mut().poll(&mut task_context),
10770            Poll::Pending
10771        ));
10772        assert!(matches!(
10773            sleep.as_mut().poll(&mut task_context),
10774            Poll::Pending
10775        ));
10776
10777        let commands = ctx.take_commands().expect("timer command");
10778        assert_eq!(
10779            commands,
10780            vec![json!({
10781                "type": "start_timer",
10782                "delay_seconds": 2,
10783            })]
10784        );
10785    }
10786
10787    #[test]
10788    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
10789        let history = vec![
10790            history_event(
10791                "TimerScheduled",
10792                json!({
10793                    "sequence": 1,
10794                    "timer_id": "timer-1",
10795                    "delay_seconds": 5,
10796                    "fire_at": "2026-07-11T12:00:05Z",
10797                }),
10798            ),
10799            history_event(
10800                "TimerFired",
10801                json!({
10802                    "sequence": 1,
10803                    "timer_id": "timer-1",
10804                    "delay_seconds": 5,
10805                    "fire_at": "2026-07-11T12:00:05Z",
10806                    "fired_at": "2026-07-11T12:00:05Z",
10807                }),
10808            ),
10809        ];
10810
10811        for _restart in 0..2 {
10812            let ctx = workflow_context(history.clone());
10813            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
10814            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10815            assert!(matches!(
10816                sleep.as_mut().poll(&mut task_context),
10817                Poll::Ready(Ok(()))
10818            ));
10819            assert!(ctx.take_commands().expect("commands").is_empty());
10820            ctx.ensure_history_consumed().expect("history consumed");
10821        }
10822    }
10823
10824    #[test]
10825    fn workflow_sleep_rejects_changed_delay_during_replay() {
10826        let ctx = workflow_context(vec![
10827            history_event(
10828                "TimerScheduled",
10829                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10830            ),
10831            history_event(
10832                "TimerFired",
10833                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10834            ),
10835        ]);
10836        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10837        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10838
10839        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10840            sleep.as_mut().poll(&mut task_context)
10841        else {
10842            panic!("changed timer delay must be rejected");
10843        };
10844        assert_eq!(failure.reason, "timer_delay_mismatch");
10845        assert_eq!(failure.sequence, Some(1));
10846    }
10847
10848    #[test]
10849    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10850        let lone_fire = WorkflowState::new(
10851            vec![history_event(
10852                "TimerFired",
10853                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10854            )],
10855            "rust-workers".to_string(),
10856            DEFAULT_CODEC.to_string(),
10857            None,
10858        )
10859        .expect_err("TimerFired requires TimerScheduled");
10860        assert!(matches!(
10861            lone_fire,
10862            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10863                if reason == "timer_schedule_missing_or_duplicate"
10864        ));
10865
10866        let wrong_identity = WorkflowState::new(
10867            vec![
10868                history_event(
10869                    "TimerScheduled",
10870                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10871                ),
10872                history_event(
10873                    "TimerFired",
10874                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10875                ),
10876            ],
10877            "rust-workers".to_string(),
10878            DEFAULT_CODEC.to_string(),
10879            None,
10880        )
10881        .expect_err("fire must match scheduled timer identity");
10882        assert!(matches!(
10883            wrong_identity,
10884            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10885                if reason == "timer_identity_mismatch"
10886        ));
10887
10888        let duplicate_fire = WorkflowState::new(
10889            vec![
10890                history_event(
10891                    "TimerScheduled",
10892                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10893                ),
10894                history_event(
10895                    "TimerFired",
10896                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10897                ),
10898                history_event(
10899                    "TimerFired",
10900                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10901                ),
10902            ],
10903            "rust-workers".to_string(),
10904            DEFAULT_CODEC.to_string(),
10905            None,
10906        )
10907        .expect_err("a durable timer cannot fire twice");
10908        assert!(matches!(
10909            duplicate_fire,
10910            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10911                if reason == "duplicate_timer_fire"
10912        ));
10913
10914        let wrong_fired_delay = WorkflowState::new(
10915            vec![
10916                history_event(
10917                    "TimerScheduled",
10918                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10919                ),
10920                history_event(
10921                    "TimerFired",
10922                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10923                ),
10924            ],
10925            "rust-workers".to_string(),
10926            DEFAULT_CODEC.to_string(),
10927            None,
10928        )
10929        .expect_err("timer schedule and fire delays must agree");
10930        assert!(matches!(
10931            wrong_fired_delay,
10932            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10933                if reason == "timer_history_delay_mismatch"
10934        ));
10935    }
10936
10937    #[test]
10938    fn replay_rejects_activity_moved_before_recorded_timer() {
10939        let ctx = workflow_context(vec![
10940            history_event(
10941                "TimerScheduled",
10942                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10943            ),
10944            history_event(
10945                "TimerFired",
10946                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10947            ),
10948            history_event(
10949                "ActivityCompleted",
10950                json!({
10951                    "sequence": 2,
10952                    "activity_type": "after-timer",
10953                    "payload_codec": DEFAULT_CODEC,
10954                    "result": fixture_envelope(json!("done")),
10955                }),
10956            ),
10957        ]);
10958        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10959        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10960
10961        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10962            activity.as_mut().poll(&mut task_context)
10963        else {
10964            panic!("reordered durable command must be rejected");
10965        };
10966        assert_eq!(failure.reason, "recorded_command_mismatch");
10967        assert_eq!(failure.sequence, Some(1));
10968        assert_eq!(failure.expected.as_deref(), Some("timer"));
10969        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10970    }
10971
10972    #[test]
10973    fn workflow_context_emits_a_typed_named_signal_wait() {
10974        let ctx = workflow_context(Vec::new());
10975        let mut signal = Box::pin(ctx.wait_signal("finish"));
10976        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10977
10978        assert!(matches!(
10979            signal.as_mut().poll(&mut task_context),
10980            Poll::Pending
10981        ));
10982        assert_eq!(
10983            ctx.take_commands().expect("signal-wait command"),
10984            vec![json!({
10985                "type": "open_signal_wait",
10986                "signal_name": "finish",
10987            })]
10988        );
10989    }
10990
10991    #[test]
10992    fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10993        let ctx = workflow_context(vec![
10994            history_event(
10995                "ConditionWaitOpened",
10996                json!({"sequence": 1, "condition_key": "signal:finish"}),
10997            ),
10998            history_event(
10999                "ConditionWaitSatisfied",
11000                json!({"sequence": 1, "condition_key": "signal:finish"}),
11001            ),
11002            history_event(
11003                "SignalReceived",
11004                json!({"signal_name": "finish", "arguments": []}),
11005            ),
11006        ]);
11007        let mut signal = Box::pin(ctx.wait_signal("finish"));
11008        let mut task_context = TaskContext::from_waker(noop_waker_ref());
11009
11010        assert!(matches!(
11011            signal.as_mut().poll(&mut task_context),
11012            Poll::Pending
11013        ));
11014        assert_eq!(
11015            ctx.take_commands().expect("typed signal-wait command"),
11016            vec![json!({
11017                "type": "open_signal_wait",
11018                "signal_name": "finish",
11019            })]
11020        );
11021    }
11022
11023    #[test]
11024    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
11025        let signal_then_timer = vec![
11026            history_event(
11027                "SignalWaitOpened",
11028                json!({"sequence": 1, "signal_name": "go"}),
11029            ),
11030            history_event(
11031                "SignalApplied",
11032                json!({
11033                    "sequence": 1,
11034                    "signal_name": "go",
11035                    "value": fixture_envelope(json!(["now"])),
11036                }),
11037            ),
11038            history_event(
11039                "TimerScheduled",
11040                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
11041            ),
11042            history_event(
11043                "TimerFired",
11044                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
11045            ),
11046        ];
11047
11048        let ctx = workflow_context(signal_then_timer.clone());
11049        let mut signal = Box::pin(ctx.wait_signal("go"));
11050        let mut task_context = TaskContext::from_waker(noop_waker_ref());
11051        assert!(matches!(
11052            signal.as_mut().poll(&mut task_context),
11053            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
11054        ));
11055        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
11056        assert!(matches!(
11057            timer.as_mut().poll(&mut task_context),
11058            Poll::Ready(Ok(()))
11059        ));
11060        ctx.ensure_history_consumed()
11061            .expect("signal and timer history consumed in order");
11062
11063        let reordered = workflow_context(signal_then_timer);
11064        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
11065        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
11066            timer_first.as_mut().poll(&mut task_context)
11067        else {
11068            panic!("timer cannot consume signal-wait-first history");
11069        };
11070        assert_eq!(failure.reason, "recorded_command_mismatch");
11071        assert_eq!(failure.sequence, Some(1));
11072        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
11073
11074        let timer_then_signal = vec![
11075            history_event(
11076                "TimerScheduled",
11077                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11078            ),
11079            history_event(
11080                "TimerFired",
11081                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11082            ),
11083            history_event(
11084                "SignalWaitOpened",
11085                json!({"sequence": 2, "signal_name": "go"}),
11086            ),
11087            history_event(
11088                "SignalApplied",
11089                json!({
11090                    "sequence": 2,
11091                    "signal_name": "go",
11092                    "value": fixture_envelope(json!([])),
11093                }),
11094            ),
11095        ];
11096        let reordered = workflow_context(timer_then_signal);
11097        let mut signal_first = Box::pin(reordered.wait_signal("go"));
11098        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
11099            signal_first.as_mut().poll(&mut task_context)
11100        else {
11101            panic!("signal wait cannot consume timer-first history");
11102        };
11103        assert_eq!(failure.reason, "recorded_command_mismatch");
11104        assert_eq!(failure.sequence, Some(1));
11105        assert_eq!(failure.expected.as_deref(), Some("timer"));
11106    }
11107
11108    #[test]
11109    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
11110        let duplicate_timer = WorkflowState::new(
11111            vec![
11112                history_event(
11113                    "TimerScheduled",
11114                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11115                ),
11116                history_event(
11117                    "TimerScheduled",
11118                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
11119                ),
11120            ],
11121            "rust-workers".to_string(),
11122            DEFAULT_CODEC.to_string(),
11123            None,
11124        )
11125        .expect_err("one workflow sequence cannot schedule two timers");
11126        assert!(matches!(
11127            duplicate_timer,
11128            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
11129                if reason == "timer_schedule_missing_or_duplicate"
11130        ));
11131
11132        let colliding_kinds = WorkflowState::new(
11133            vec![
11134                history_event(
11135                    "TimerScheduled",
11136                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11137                ),
11138                history_event(
11139                    "ActivityCompleted",
11140                    json!({"sequence": 1, "activity_type": "same-sequence"}),
11141                ),
11142            ],
11143            "rust-workers".to_string(),
11144            DEFAULT_CODEC.to_string(),
11145            None,
11146        )
11147        .expect_err("one workflow sequence cannot identify two command kinds");
11148        assert!(matches!(
11149            colliding_kinds,
11150            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
11151                if reason == "durable_command_sequence_collision"
11152        ));
11153
11154        let duplicate_signal_wait = WorkflowState::new(
11155            vec![
11156                history_event(
11157                    "SignalWaitOpened",
11158                    json!({"sequence": 1, "signal_name": "go"}),
11159                ),
11160                history_event(
11161                    "SignalWaitOpened",
11162                    json!({"sequence": 1, "signal_name": "go"}),
11163                ),
11164            ],
11165            "rust-workers".to_string(),
11166            DEFAULT_CODEC.to_string(),
11167            None,
11168        )
11169        .expect_err("one workflow sequence cannot open two signal waits");
11170        assert!(matches!(
11171            duplicate_signal_wait,
11172            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
11173                if reason == "signal_wait_open_missing_or_duplicate"
11174        ));
11175    }
11176
11177    #[test]
11178    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
11179        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
11180            .expect("side-effect result");
11181        let ctx = workflow_context(vec![history_event(
11182            "SideEffectRecorded",
11183            json!({"sequence": 99, "result": result}),
11184        )]);
11185
11186        let replayed: Value = ctx
11187            .side_effect(|| panic!("recorded side effect must not run"))
11188            .expect("positive global workflow sequence is valid");
11189        assert_eq!(replayed, json!({"captured": true}));
11190        ctx.ensure_history_consumed().expect("history consumed");
11191    }
11192
11193    #[test]
11194    fn workflow_history_rejects_zero_and_descending_command_sequences() {
11195        let result =
11196            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
11197        let zero = WorkflowState::new(
11198            vec![history_event(
11199                "SideEffectRecorded",
11200                json!({"sequence": 0, "result": result.clone()}),
11201            )],
11202            "rust-workers".to_string(),
11203            DEFAULT_CODEC.to_string(),
11204            None,
11205        )
11206        .expect_err("durable command sequences must be positive");
11207        assert!(matches!(
11208            zero,
11209            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
11210                if reason == "durable_command_sequence_invalid"
11211        ));
11212
11213        let descending = WorkflowState::new(
11214            vec![
11215                history_event(
11216                    "SideEffectRecorded",
11217                    json!({"sequence": 3, "result": result}),
11218                ),
11219                history_event(
11220                    "VersionMarkerRecorded",
11221                    json!({
11222                        "sequence": 2,
11223                        "change_id": "descending-marker",
11224                        "version": 1,
11225                        "min_supported": 1,
11226                        "max_supported": 1,
11227                    }),
11228                ),
11229            ],
11230            "rust-workers".to_string(),
11231            DEFAULT_CODEC.to_string(),
11232            None,
11233        )
11234        .expect_err("new durable commands must remain strictly ordered");
11235        let Error::NonDeterministicReplay(failure) = descending else {
11236            panic!("expected typed replay failure");
11237        };
11238        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
11239        assert_eq!(failure.sequence, Some(2));
11240        assert_eq!(
11241            failure.expected.as_deref(),
11242            Some("workflow sequence greater than 3")
11243        );
11244        assert_eq!(failure.actual.as_deref(), Some("2"));
11245    }
11246
11247    #[test]
11248    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
11249        fn worker() -> Worker {
11250            let client = Client::new("http://127.0.0.1:8080").expect("client");
11251            let mut worker = Worker::new(client, "rust-workers");
11252            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
11253                ctx.wait_signal("finish").await?;
11254                let marker: String =
11255                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
11256                assert_eq!(marker, "after-finish");
11257                Ok(json!("finished"))
11258            });
11259            worker
11260        }
11261
11262        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
11263            .expect("side-effect result");
11264        let task = workflow_task(
11265            "rust.finish-after-gaps",
11266            vec![
11267                history_event(
11268                    "SignalWaitOpened",
11269                    json!({"sequence": 1, "signal_name": "finish"}),
11270                ),
11271                history_event(
11272                    "SignalReceived",
11273                    json!({
11274                        "signal_id": "increment-3",
11275                        "signal_name": "increment",
11276                        "workflow_sequence": 2,
11277                        "payload_codec": DEFAULT_CODEC,
11278                        "arguments": fixture_envelope(json!([3])),
11279                    }),
11280                ),
11281                history_event(
11282                    "SignalReceived",
11283                    json!({
11284                        "signal_id": "increment-5",
11285                        "signal_name": "increment",
11286                        "workflow_sequence": 3,
11287                        "payload_codec": DEFAULT_CODEC,
11288                        "arguments": fixture_envelope(json!([5])),
11289                    }),
11290                ),
11291                history_event(
11292                    "SignalReceived",
11293                    json!({
11294                        "signal_id": "finish",
11295                        "signal_name": "finish",
11296                        "workflow_sequence": 4,
11297                        "payload_codec": DEFAULT_CODEC,
11298                        "arguments": fixture_envelope(json!([])),
11299                    }),
11300                ),
11301                history_event(
11302                    "SignalApplied",
11303                    json!({
11304                        "sequence": 1,
11305                        "signal_id": "finish",
11306                        "signal_name": "finish",
11307                        "payload_codec": DEFAULT_CODEC,
11308                        "value": fixture_envelope(json!([])),
11309                    }),
11310                ),
11311                history_event(
11312                    "SideEffectRecorded",
11313                    json!({"sequence": 5, "result": marker}),
11314                ),
11315            ],
11316            DEFAULT_CODEC,
11317        );
11318
11319        for _original_or_cold_worker in 0..2 {
11320            let commands = worker()
11321                .execute_workflow_task(task.clone())
11322                .expect("signal gaps preserve deterministic replay");
11323            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
11324            assert_eq!(commands[0]["type"], "complete_workflow");
11325            assert_eq!(
11326                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
11327                json!("finished")
11328            );
11329        }
11330    }
11331
11332    #[test]
11333    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
11334        let ctx = workflow_context(Vec::new());
11335        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
11336        let mut task_context = TaskContext::from_waker(noop_waker_ref());
11337        assert!(matches!(
11338            sleep.as_mut().poll(&mut task_context),
11339            Poll::Ready(Err(Error::TimerDurationOverflow))
11340        ));
11341        assert!(ctx.take_commands().expect("commands").is_empty());
11342    }
11343
11344    #[test]
11345    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
11346        let client = Client::new("http://127.0.0.1:8080").expect("client");
11347        let mut worker = Worker::new(client, "rust-workers");
11348        worker.register_workflow("rust.timer", |ctx, _input| async move {
11349            ctx.sleep(Duration::from_secs(5)).await?;
11350            ctx.activity("after-timer", json!([])).await
11351        });
11352
11353        let task = |history_events| WorkflowTask {
11354            task_id: "wft-rust-timer-1".to_string(),
11355            workflow_id: Some("wf-rust-timer".to_string()),
11356            run_id: Some("run-rust-timer".to_string()),
11357            workflow_type: "rust.timer".to_string(),
11358            payload_codec: DEFAULT_CODEC.to_string(),
11359            arguments: Some(
11360                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11361            ),
11362            history_events,
11363            total_history_events: None,
11364            history_size_bytes: None,
11365            continue_as_new_recommended: None,
11366            history_budget_pressure: None,
11367            next_history_page_token: None,
11368            workflow_task_attempt: 1,
11369            workflow_signal_id: None,
11370            signal_name: None,
11371            signal_arguments: None,
11372            workflow_update_id: None,
11373            update_name: None,
11374            lease_owner: Some("rust-worker".to_string()),
11375        };
11376
11377        let initial = worker
11378            .execute_workflow_task(task(Vec::new()))
11379            .expect("initial timer task");
11380        assert_eq!(
11381            initial,
11382            vec![json!({"type": "start_timer", "delay_seconds": 5})]
11383        );
11384
11385        let activity_result =
11386            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
11387        let replayed = worker
11388            .execute_workflow_task(task(vec![
11389                history_event(
11390                    "TimerScheduled",
11391                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11392                ),
11393                history_event(
11394                    "TimerFired",
11395                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11396                ),
11397                history_event(
11398                    "ActivityCompleted",
11399                    json!({
11400                        "sequence": 2,
11401                        "activity_type": "after-timer",
11402                        "payload_codec": DEFAULT_CODEC,
11403                        "result": activity_result,
11404                    }),
11405                ),
11406            ]))
11407            .expect("replayed workflow task");
11408        assert_eq!(replayed.len(), 1);
11409        assert_eq!(replayed[0]["type"], "complete_workflow");
11410        assert_eq!(
11411            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
11412            json!("done")
11413        );
11414    }
11415
11416    #[test]
11417    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
11418        let client = Client::new("http://127.0.0.1:8080").expect("client");
11419        let mut worker = Worker::new(client, "rust-workers");
11420        worker.register_workflow("rust.continue", |ctx, _input| async move {
11421            ctx.continue_as_new_with_options(
11422                ContinueAsNewOptions::new()
11423                    .workflow_type("rust.next")
11424                    .task_queue("next-workers"),
11425                json!([2, {"cursor": "next"}]),
11426            )
11427        });
11428
11429        let commands = worker
11430            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
11431            .expect("continue-as-new command");
11432
11433        assert_eq!(commands.len(), 1);
11434        assert_eq!(commands[0]["type"], "continue_as_new");
11435        assert_eq!(commands[0]["workflow_type"], "rust.next");
11436        assert_eq!(commands[0]["queue"], "next-workers");
11437        assert_eq!(
11438            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
11439                .expect("continue-as-new arguments"),
11440            json!([2, {"cursor": "next"}])
11441        );
11442    }
11443
11444    #[test]
11445    fn continue_as_new_preserves_typed_arguments() {
11446        let client = Client::new("http://127.0.0.1:8080").expect("client");
11447        let mut worker = Worker::new(client, "rust-workers");
11448        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
11449            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
11450            unreachable!("continue-as-new returns a control-flow error")
11451        });
11452
11453        let commands = worker
11454            .execute_workflow_task(workflow_task(
11455                "rust.typed-continue",
11456                Vec::new(),
11457                DEFAULT_CODEC,
11458            ))
11459            .expect("typed continue-as-new command");
11460
11461        assert_eq!(commands[0]["type"], "continue_as_new");
11462        assert_eq!(
11463            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
11464                .expect("typed continue arguments"),
11465            AvroValue::Array(vec![typed_fidelity_probe()])
11466        );
11467    }
11468
11469    #[test]
11470    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
11471        let client = Client::new("http://127.0.0.1:8080").expect("client");
11472        let mut worker = Worker::new(client, "rust-workers");
11473        worker.register_workflow("rust.continue", |ctx, _input| async move {
11474            ctx.continue_as_new(json!([2]))
11475        });
11476        let task = workflow_task(
11477            "rust.continue",
11478            vec![history_event(
11479                "WorkflowContinuedAsNew",
11480                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
11481            )],
11482            DEFAULT_CODEC,
11483        );
11484
11485        for _worker_restart_or_redelivery in 0..2 {
11486            let commands = worker
11487                .execute_workflow_task(task.clone())
11488                .expect("recorded transition replays");
11489            assert!(
11490                commands.is_empty(),
11491                "replay must not emit another successor"
11492            );
11493        }
11494    }
11495
11496    #[test]
11497    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
11498        let ctx = workflow_context(Vec::new());
11499        let error = ctx
11500            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
11501            .expect_err("blank queue must be rejected");
11502
11503        let Error::InvalidContinueAsNewOptions(error) = error else {
11504            panic!("expected typed continue-as-new validation error");
11505        };
11506        assert_eq!(error.field, "task_queue");
11507        assert!(ctx.take_commands().expect("commands").is_empty());
11508    }
11509
11510    #[test]
11511    fn workflow_context_exposes_server_history_budget() {
11512        let client = Client::new("http://127.0.0.1:8080").expect("client");
11513        let mut worker = Worker::new(client, "rust-workers");
11514        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
11515            let budget = ctx.history_budget()?;
11516            Ok(json!({
11517                "events": budget.event_count,
11518                "bytes": budget.size_bytes,
11519                "recommended": budget.continue_as_new_recommended,
11520                "pressure": budget.pressure,
11521            }))
11522        });
11523        let task: WorkflowTask = serde_json::from_value(json!({
11524            "task_id": "task-history-budget",
11525            "workflow_type": "rust.history-budget",
11526            "payload_codec": DEFAULT_CODEC,
11527            "history_events": [],
11528            "total_history_events": 480,
11529            "history_size_bytes": 1_048_576,
11530            "continue_as_new_recommended": true,
11531            "history_budget_pressure": "continue_as_new_recommended",
11532        }))
11533        .expect("published workflow task");
11534
11535        let commands = worker
11536            .execute_workflow_task(task)
11537            .expect("history-budget workflow");
11538        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
11539        assert_eq!(result["events"], 480);
11540        assert_eq!(result["bytes"], 1_048_576);
11541        assert_eq!(result["recommended"], true);
11542        assert_eq!(result["pressure"], "continue_as_new_recommended");
11543    }
11544
11545    #[test]
11546    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
11547        let client = Client::new("http://127.0.0.1:8080").expect("client");
11548        let mut worker = Worker::new(client, "rust-workers");
11549        worker.register_workflow("rust.failing", |_ctx, _input| async move {
11550            Err(Error::Codec("rust_conformance_failure".to_string()))
11551        });
11552        let task = WorkflowTask {
11553            task_id: "wft-rust-failing-1".to_string(),
11554            workflow_id: Some("wf-rust-failing".to_string()),
11555            run_id: Some("run-rust-failing".to_string()),
11556            workflow_type: "rust.failing".to_string(),
11557            payload_codec: DEFAULT_CODEC.to_string(),
11558            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11559            history_events: Vec::new(),
11560            total_history_events: Some(0),
11561            history_size_bytes: None,
11562            continue_as_new_recommended: None,
11563            history_budget_pressure: None,
11564            next_history_page_token: None,
11565            workflow_task_attempt: 1,
11566            workflow_signal_id: None,
11567            signal_name: None,
11568            signal_arguments: None,
11569            workflow_update_id: None,
11570            update_name: None,
11571            lease_owner: Some("rust-worker".to_string()),
11572        };
11573
11574        let commands = worker
11575            .execute_workflow_task(task)
11576            .expect("handler failure becomes a workflow command");
11577
11578        assert_eq!(commands.len(), 1);
11579        assert_eq!(commands[0]["type"], "fail_workflow");
11580        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
11581        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
11582        assert_eq!(commands[0]["non_retryable"], false);
11583        assert_eq!(
11584            commands[0]["message"],
11585            "codec error: rust_conformance_failure"
11586        );
11587        assert_eq!(
11588            commands[0]["exception"]["message"],
11589            "codec error: rust_conformance_failure"
11590        );
11591    }
11592
11593    #[test]
11594    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
11595        let client = Client::new("http://127.0.0.1:8080").expect("client");
11596        let mut worker = Worker::new(client, "rust-workers");
11597        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
11598            let _: String = ctx.side_effect(|| "captured".to_string())?;
11599            Err(Error::WorkerLoop("application failure".to_string()))
11600        });
11601
11602        let commands = worker
11603            .execute_workflow_task(workflow_task(
11604                "rust.failing-after-side-effect",
11605                Vec::new(),
11606                DEFAULT_CODEC,
11607            ))
11608            .expect("ordinary failure remains a workflow decision");
11609
11610        assert_eq!(commands.len(), 2);
11611        assert_eq!(commands[0]["type"], "record_side_effect");
11612        assert_eq!(commands[1]["type"], "fail_workflow");
11613    }
11614
11615    #[test]
11616    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
11617        let client = Client::new("http://127.0.0.1:8080").expect("client");
11618        let mut worker = Worker::new(client, "rust-workers");
11619        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
11620            Err(Error::WorkerLoop("application failure".to_string()))
11621        });
11622        let result =
11623            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
11624
11625        let error = worker
11626            .execute_workflow_task(workflow_task(
11627                "rust.removed-side-effect",
11628                vec![history_event(
11629                    "SideEffectRecorded",
11630                    json!({"sequence": 1, "result": result}),
11631                )],
11632                DEFAULT_CODEC,
11633            ))
11634            .expect_err("removed committed history must not become fail_workflow");
11635
11636        let Error::NonDeterministicReplay(failure) = error else {
11637            panic!("expected typed replay failure");
11638        };
11639        assert_eq!(failure.reason, "recorded_commands_unconsumed");
11640        assert_eq!(failure.sequence, Some(1));
11641        assert_eq!(failure.expected.as_deref(), Some("side effect"));
11642    }
11643
11644    #[test]
11645    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
11646        let client = Client::new("http://127.0.0.1:8080").expect("client");
11647        let mut worker = Worker::new(client, "rust-workers");
11648        worker.register_workflow(
11649            "rust.side-effect-before-marker-error",
11650            |ctx, _input| async move {
11651                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
11652                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
11653                ctx.get_version("restart-safe", 2, 2)?;
11654                Ok(Value::Null)
11655            },
11656        );
11657
11658        let error = worker
11659            .execute_workflow_task(workflow_task(
11660                "rust.side-effect-before-marker-error",
11661                vec![history_event(
11662                    "VersionMarkerRecorded",
11663                    json!({
11664                        "sequence": 1,
11665                        "change_id": "restart-safe",
11666                        "version": 1,
11667                        "min_supported": 1,
11668                        "max_supported": 1,
11669                    }),
11670                )],
11671                DEFAULT_CODEC,
11672            ))
11673            .expect_err("replay error must return no queued workflow commands");
11674
11675        let Error::NonDeterministicReplay(failure) = error else {
11676            panic!("expected typed replay failure");
11677        };
11678        assert_eq!(failure.reason, "version_marker_incompatible_range");
11679        assert_eq!(failure.sequence, Some(1));
11680    }
11681
11682    #[test]
11683    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
11684        let client = Client::new("http://127.0.0.1:8080").expect("client");
11685        let mut worker = Worker::new(client, "rust-workers");
11686        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
11687            ctx.sleep(Duration::from_secs(5)).await?;
11688            Ok(json!({"status": "timer fired"}))
11689        });
11690
11691        let task = WorkflowTask {
11692            task_id: "wft-rust-timer-pending".to_string(),
11693            workflow_id: Some("wf-rust-timer".to_string()),
11694            run_id: Some("run-rust-timer".to_string()),
11695            workflow_type: "rust.timer.pending".to_string(),
11696            payload_codec: DEFAULT_CODEC.to_string(),
11697            arguments: Some(
11698                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11699            ),
11700            history_events: vec![history_event(
11701                "TimerScheduled",
11702                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11703            )],
11704            total_history_events: Some(1),
11705            history_size_bytes: None,
11706            continue_as_new_recommended: None,
11707            history_budget_pressure: None,
11708            next_history_page_token: None,
11709            workflow_task_attempt: 1,
11710            workflow_signal_id: None,
11711            signal_name: None,
11712            signal_arguments: None,
11713            workflow_update_id: None,
11714            update_name: None,
11715            lease_owner: Some("rust-worker".to_string()),
11716        };
11717
11718        for _redelivery_or_restart in 0..2 {
11719            let commands = worker
11720                .execute_workflow_task(task.clone())
11721                .expect("recorded timer remains pending");
11722            assert!(
11723                commands.is_empty(),
11724                "recorded timer must not be rescheduled"
11725            );
11726        }
11727    }
11728
11729    #[test]
11730    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
11731        let client = Client::new("http://127.0.0.1:8080").expect("client");
11732        let mut worker = Worker::new(client, "rust-workers");
11733        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
11734            Ok(json!({"status": "completed"}))
11735        });
11736        let task = WorkflowTask {
11737            task_id: "wft-rust-timer-removed".to_string(),
11738            workflow_id: Some("wf-rust-timer".to_string()),
11739            run_id: Some("run-rust-timer".to_string()),
11740            workflow_type: "rust.timer.removed".to_string(),
11741            payload_codec: DEFAULT_CODEC.to_string(),
11742            arguments: Some(
11743                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11744            ),
11745            history_events: vec![
11746                history_event(
11747                    "TimerScheduled",
11748                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11749                ),
11750                history_event(
11751                    "TimerFired",
11752                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
11753                ),
11754            ],
11755            total_history_events: Some(2),
11756            history_size_bytes: None,
11757            continue_as_new_recommended: None,
11758            history_budget_pressure: None,
11759            next_history_page_token: None,
11760            workflow_task_attempt: 1,
11761            workflow_signal_id: None,
11762            signal_name: None,
11763            signal_arguments: None,
11764            workflow_update_id: None,
11765            update_name: None,
11766            lease_owner: Some("rust-worker".to_string()),
11767        };
11768
11769        let Error::NonDeterministicReplay(failure) = worker
11770            .execute_workflow_task(task)
11771            .expect_err("removed timer must fail replay")
11772        else {
11773            panic!("expected typed replay failure");
11774        };
11775        assert_eq!(failure.reason, "recorded_commands_unconsumed");
11776        assert_eq!(failure.sequence, Some(1));
11777    }
11778
11779    #[test]
11780    fn workflow_context_emits_explicit_child_workflow_contract() {
11781        let ctx = WorkflowContext {
11782            state: Arc::new(Mutex::new(
11783                WorkflowState::new_with_identity(
11784                    Vec::new(),
11785                    Some("wf-parent".to_string()),
11786                    Some("run-parent".to_string()),
11787                    "parent-workers".to_string(),
11788                    DEFAULT_CODEC.to_string(),
11789                    None,
11790                )
11791                .expect("workflow state"),
11792            )),
11793        };
11794        let options = ChildWorkflowOptions::new("python-workers")
11795            .parent_close_policy(ParentClosePolicy::RequestCancel)
11796            .retry_policy(ChildWorkflowRetryPolicy {
11797                max_attempts: Some(3),
11798                backoff_seconds: vec![1, 5],
11799                non_retryable_error_types: vec!["ValidationError".to_string()],
11800            })
11801            .execution_timeout_seconds(600)
11802            .run_timeout_seconds(120);
11803        let mut call = Box::pin(ctx.start_child_workflow(
11804            "python.fulfil-order",
11805            options,
11806            json!([{"order_id": "order-42"}]),
11807        ));
11808        let mut task_context = TaskContext::from_waker(noop_waker_ref());
11809
11810        assert!(matches!(
11811            call.as_mut().poll(&mut task_context),
11812            Poll::Pending
11813        ));
11814        let commands = ctx.take_commands().expect("commands");
11815        assert_eq!(commands.len(), 1);
11816        let command = &commands[0];
11817        assert_eq!(command["type"], "start_child_workflow");
11818        assert_eq!(command["workflow_type"], "python.fulfil-order");
11819        assert_eq!(command["queue"], "python-workers");
11820        assert_eq!(command["parent_close_policy"], "request_cancel");
11821        assert_eq!(command["retry_policy"]["max_attempts"], 3);
11822        assert_eq!(command["execution_timeout_seconds"], 600);
11823        assert_eq!(command["run_timeout_seconds"], 120);
11824        assert_eq!(
11825            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
11826            json!([{"order_id": "order-42"}])
11827        );
11828    }
11829
11830    fn child_parent_worker() -> Worker {
11831        let client = Client::new("http://127.0.0.1:8080").expect("client");
11832        let mut worker = Worker::new(client, "rust-parent-workers");
11833        worker.register_workflow("rust.parent", |ctx, _input| async move {
11834            let child = ctx
11835                .start_child_workflow(
11836                    "python.child",
11837                    ChildWorkflowOptions::new("python-child-workers")
11838                        .parent_close_policy(ParentClosePolicy::Terminate),
11839                    json!([{"codec_probe": [1, true, "rust"]}]),
11840                )
11841                .await?;
11842            Ok(json!({
11843                "parent_workflow_id": child.parent.workflow_id,
11844                "parent_run_id": child.parent.run_id,
11845                "child_workflow_id": child.child.workflow_id,
11846                "child_run_id": child.child.run_id,
11847                "child_workflow_type": child.child_workflow_type,
11848                "result": child.result,
11849            }))
11850        });
11851        worker
11852    }
11853
11854    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11855        WorkflowTask {
11856            task_id: "wft-child-parent".to_string(),
11857            workflow_id: Some("wf-parent".to_string()),
11858            run_id: Some("run-parent".to_string()),
11859            workflow_type: "rust.parent".to_string(),
11860            payload_codec: DEFAULT_CODEC.to_string(),
11861            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11862            history_events: vec![
11863                HistoryEvent {
11864                    event_type: "ChildWorkflowScheduled".to_string(),
11865                    payload: json!({
11866                        "sequence": 1,
11867                        "child_call_id": "call-child",
11868                        "child_workflow_instance_id": "wf-child",
11869                        "child_workflow_run_id": "run-child",
11870                        "child_workflow_type": "python.child",
11871                    }),
11872                    raw: HashMap::new(),
11873                },
11874                HistoryEvent {
11875                    event_type: event_type.to_string(),
11876                    payload,
11877                    raw: HashMap::new(),
11878                },
11879            ],
11880            total_history_events: Some(2),
11881            history_size_bytes: None,
11882            continue_as_new_recommended: None,
11883            history_budget_pressure: None,
11884            next_history_page_token: None,
11885            workflow_task_attempt: 1,
11886            workflow_signal_id: None,
11887            signal_name: None,
11888            signal_arguments: None,
11889            workflow_update_id: None,
11890            update_name: None,
11891            lease_owner: Some("rust-worker".to_string()),
11892        }
11893    }
11894
11895    #[test]
11896    fn committed_child_result_replays_without_starting_a_duplicate() {
11897        let worker = child_parent_worker();
11898        let task = child_parent_task(
11899            "ChildRunCompleted",
11900            json!({
11901                "sequence": 1,
11902                "child_call_id": "call-child",
11903                "child_workflow_instance_id": "wf-child",
11904                "child_workflow_run_id": "run-child",
11905                "child_workflow_type": "python.child",
11906                "payload_codec": DEFAULT_CODEC,
11907                "result": fixture_envelope(json!({"from":"python","ok":true})),
11908            }),
11909        );
11910
11911        for _restart in 0..2 {
11912            let commands = worker
11913                .execute_workflow_task(task.clone())
11914                .expect("replayed parent task");
11915            assert_eq!(commands.len(), 1);
11916            assert_eq!(commands[0]["type"], "complete_workflow");
11917            assert!(!commands
11918                .iter()
11919                .any(|command| command["type"] == "start_child_workflow"));
11920            let output =
11921                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
11922            assert_eq!(output["parent_workflow_id"], "wf-parent");
11923            assert_eq!(output["parent_run_id"], "run-parent");
11924            assert_eq!(output["child_workflow_id"], "wf-child");
11925            assert_eq!(output["child_run_id"], "run-child");
11926            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11927        }
11928    }
11929
11930    #[test]
11931    fn typed_child_arguments_and_results_survive_replay() {
11932        let client = Client::new("http://127.0.0.1:8080").expect("client");
11933        let mut worker = Worker::new(client, "rust-parent-workers");
11934        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11935            let child = ctx
11936                .start_child_workflow_avro_value(
11937                    "python.typed-child",
11938                    ChildWorkflowOptions::new("python-workers"),
11939                    AvroValue::Array(vec![typed_fidelity_probe()]),
11940                )
11941                .await?;
11942            Ok(child.result)
11943        });
11944
11945        let initial = worker
11946            .execute_workflow_task(workflow_task(
11947                "rust.typed-parent",
11948                Vec::new(),
11949                DEFAULT_CODEC,
11950            ))
11951            .expect("typed child start");
11952        assert_eq!(initial[0]["type"], "start_child_workflow");
11953        assert_eq!(
11954            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11955                .expect("typed child arguments"),
11956            AvroValue::Array(vec![typed_fidelity_probe()])
11957        );
11958
11959        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11960            .expect("typed child result");
11961        let task = workflow_task(
11962            "rust.typed-parent",
11963            vec![
11964                history_event(
11965                    "ChildWorkflowScheduled",
11966                    json!({
11967                        "sequence": 1,
11968                        "child_call_id": "call-typed",
11969                        "child_workflow_instance_id": "wf-child",
11970                        "child_workflow_run_id": "run-child",
11971                        "child_workflow_type": "python.typed-child",
11972                    }),
11973                ),
11974                history_event(
11975                    "ChildRunCompleted",
11976                    json!({
11977                        "sequence": 1,
11978                        "child_call_id": "call-typed",
11979                        "child_workflow_instance_id": "wf-child",
11980                        "child_workflow_run_id": "run-child",
11981                        "child_workflow_type": "python.typed-child",
11982                        "payload_codec": DEFAULT_CODEC,
11983                        "result": result,
11984                    }),
11985                ),
11986            ],
11987            DEFAULT_CODEC,
11988        );
11989
11990        let commands = worker
11991            .execute_workflow_task(task)
11992            .expect("typed child replay");
11993        assert_eq!(commands[0]["type"], "complete_workflow");
11994        assert_eq!(
11995            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11996                .expect("typed parent result"),
11997            typed_fidelity_probe()
11998        );
11999    }
12000
12001    #[test]
12002    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
12003        let worker = child_parent_worker();
12004        let mut task = child_parent_task("unused", Value::Null);
12005        task.history_events.truncate(1);
12006        task.total_history_events = Some(1);
12007
12008        for _redelivery_or_restart in 0..2 {
12009            let commands = worker
12010                .execute_workflow_task(task.clone())
12011                .expect("recorded child remains pending");
12012            assert!(
12013                commands.is_empty(),
12014                "recorded pending child must not be started again"
12015            );
12016        }
12017    }
12018
12019    #[test]
12020    fn child_cancellation_becomes_stable_parent_failure_command() {
12021        let worker = child_parent_worker();
12022        let task = child_parent_task(
12023            "ChildRunCancelled",
12024            json!({
12025                "sequence": 1,
12026                "child_workflow_instance_id": "wf-child",
12027                "child_workflow_run_id": "run-child",
12028                "child_workflow_type": "python.child",
12029                "failure_id": "failure-child",
12030                "failure_category": "cancelled",
12031                "message": "cancelled by parent-close policy",
12032            }),
12033        );
12034
12035        let commands = worker
12036            .execute_workflow_task(task)
12037            .expect("parent settlement");
12038        assert_eq!(commands.len(), 1);
12039        assert_eq!(commands[0]["type"], "fail_workflow");
12040        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
12041        assert_eq!(
12042            commands[0]["exception"]["properties"]["reason"],
12043            "cancelled"
12044        );
12045        assert_eq!(
12046            commands[0]["exception"]["properties"]["child_workflow_run_id"],
12047            "run-child"
12048        );
12049    }
12050
12051    #[test]
12052    fn workflow_can_handle_typed_child_failure() {
12053        let client = Client::new("http://127.0.0.1:8080").expect("client");
12054        let mut worker = Worker::new(client, "rust-parent-workers");
12055        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
12056            match ctx
12057                .start_child_workflow(
12058                    "python.child",
12059                    ChildWorkflowOptions::new("python-child-workers"),
12060                    json!([]),
12061                )
12062                .await
12063            {
12064                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
12065                    "reason": failure.reason,
12066                    "failure_id": failure.failure_id,
12067                    "exception_class": failure.exception_class,
12068                    "child_run_id": failure.child_workflow_run_id,
12069                })),
12070                Err(error) => Err(error),
12071                Ok(_) => Err(Error::WorkerLoop(
12072                    "child unexpectedly succeeded".to_string(),
12073                )),
12074            }
12075        });
12076        let mut task = child_parent_task(
12077            "ChildRunFailed",
12078            json!({
12079                "sequence": 1,
12080                "child_workflow_instance_id": "wf-child",
12081                "child_workflow_run_id": "run-child",
12082                "child_workflow_type": "python.child",
12083                "failure_id": "failure-child",
12084                "failure_category": "child_workflow",
12085                "message": "payment rejected",
12086                "exception": {
12087                    "type": "PaymentRejected",
12088                    "class": "payments.PaymentRejected",
12089                    "message": "payment rejected"
12090                }
12091            }),
12092        );
12093        task.workflow_type = "rust.handled-parent".to_string();
12094
12095        let commands = worker.execute_workflow_task(task).expect("handled failure");
12096        assert_eq!(commands[0]["type"], "complete_workflow");
12097        let output =
12098            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
12099        assert_eq!(output["reason"], "child_workflow");
12100        assert_eq!(output["failure_id"], "failure-child");
12101        assert_eq!(output["exception_class"], "payments.PaymentRejected");
12102        assert_eq!(output["child_run_id"], "run-child");
12103    }
12104
12105    #[test]
12106    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
12107        let client = Client::new("http://127.0.0.1:8080").expect("client");
12108        let mut worker = Worker::new(client, "rust-workers");
12109
12110        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
12111            let signal = ctx.wait_signal("start").await?;
12112            let name = signal
12113                .first()
12114                .and_then(|value| value.as_str())
12115                .unwrap_or("world");
12116            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
12117            Ok(json!({
12118                "greeting": greeting,
12119                "language": "rust"
12120            }))
12121        });
12122
12123        let signal_arguments =
12124            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
12125        let task = WorkflowTask {
12126            task_id: "wft-rust-signal-1".to_string(),
12127            workflow_id: Some("wf-rust-hello".to_string()),
12128            run_id: Some("run-rust-hello".to_string()),
12129            workflow_type: "rust.hello_workflow".to_string(),
12130            payload_codec: DEFAULT_CODEC.to_string(),
12131            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
12132            history_events: vec![HistoryEvent {
12133                event_type: "SignalReceived".to_string(),
12134                payload: json!({
12135                    "signal_id": "sig-rust-1",
12136                    "signal_name": "start"
12137                }),
12138                raw: HashMap::new(),
12139            }],
12140            total_history_events: Some(1),
12141            history_size_bytes: None,
12142            continue_as_new_recommended: None,
12143            history_budget_pressure: None,
12144            next_history_page_token: None,
12145            workflow_task_attempt: 1,
12146            workflow_signal_id: Some("sig-rust-1".to_string()),
12147            signal_name: Some("start".to_string()),
12148            signal_arguments: Some(signal_arguments),
12149            workflow_update_id: None,
12150            update_name: None,
12151            lease_owner: Some("rust-worker".to_string()),
12152        };
12153
12154        let commands = worker.execute_workflow_task(task).expect("workflow task");
12155
12156        assert_eq!(commands.len(), 1);
12157        assert_eq!(commands[0]["type"], "schedule_activity");
12158        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
12159        assert_eq!(
12160            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
12161            json!(["Rust"])
12162        );
12163    }
12164
12165    #[test]
12166    fn workflow_task_appends_paginated_history_events() {
12167        let mut task = WorkflowTask {
12168            task_id: "wft-rust-pages-1".to_string(),
12169            workflow_id: Some("wf-rust-pages".to_string()),
12170            run_id: Some("run-rust-pages".to_string()),
12171            workflow_type: "rust.hello_workflow".to_string(),
12172            payload_codec: DEFAULT_CODEC.to_string(),
12173            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
12174            history_events: vec![HistoryEvent {
12175                event_type: "WorkflowStarted".to_string(),
12176                payload: json!({}),
12177                raw: HashMap::new(),
12178            }],
12179            total_history_events: Some(3),
12180            history_size_bytes: None,
12181            continue_as_new_recommended: None,
12182            history_budget_pressure: None,
12183            next_history_page_token: Some("MQ==".to_string()),
12184            workflow_task_attempt: 1,
12185            workflow_signal_id: None,
12186            signal_name: None,
12187            signal_arguments: None,
12188            workflow_update_id: None,
12189            update_name: None,
12190            lease_owner: Some("rust-worker".to_string()),
12191        };
12192
12193        task.append_history_page(WorkflowTaskHistoryPage {
12194            history_events: vec![
12195                HistoryEvent {
12196                    event_type: "SignalReceived".to_string(),
12197                    payload: json!({
12198                        "signal_id": "sig-rust-1",
12199                        "signal_name": "start",
12200                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
12201                            .expect("signal arguments")
12202                    }),
12203                    raw: HashMap::new(),
12204                },
12205                HistoryEvent {
12206                    event_type: "MarkerRecorded".to_string(),
12207                    payload: json!({"sequence": 3}),
12208                    raw: HashMap::new(),
12209                },
12210            ],
12211            total_history_events: Some(3),
12212            next_history_page_token: None,
12213        });
12214
12215        assert_eq!(task.history_events.len(), 3);
12216        assert_eq!(task.total_history_events, Some(3));
12217        assert_eq!(task.next_history_page_token, None);
12218
12219        let signal = task
12220            .history_events
12221            .iter()
12222            .find(|event| event.event_type == "SignalReceived")
12223            .expect("signal event");
12224        assert_eq!(
12225            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
12226            vec![AvroValue::String("Rust".to_string())]
12227        );
12228    }
12229
12230    #[tokio::test]
12231    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
12232        let client = Client::new("http://127.0.0.1:8080").expect("client");
12233        let mut worker = Worker::new(client, "rust-workers");
12234        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12235        worker.register_query("counter", "current", |ctx, _args| async move {
12236            let mut count = 0_i64;
12237            for signal in ctx.signal_events() {
12238                let value = signal
12239                    .arguments
12240                    .first()
12241                    .and_then(Value::as_i64)
12242                    .unwrap_or_default();
12243                match signal.name.as_str() {
12244                    "increment" => count += value,
12245                    "set" => count = value,
12246                    _ => {}
12247                }
12248            }
12249            Ok(json!(count))
12250        });
12251
12252        let task = QueryTask {
12253            query_task_id: "query-rust-counter".to_string(),
12254            query_task_attempt: 1,
12255            lease_owner: Some("rust-worker".to_string()),
12256            workflow_id: Some("counter-1".to_string()),
12257            run_id: Some("run-counter-1".to_string()),
12258            workflow_type: "counter".to_string(),
12259            query_name: "current".to_string(),
12260            payload_codec: DEFAULT_CODEC.to_string(),
12261            workflow_arguments: Some(
12262                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
12263            ),
12264            query_arguments: Some(
12265                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
12266            ),
12267            history_events: vec![
12268                HistoryEvent {
12269                    event_type: "SignalReceived".to_string(),
12270                    payload: json!({
12271                        "signal_id": "php-signal-1",
12272                        "signal_name": "increment",
12273                        "workflow_sequence": 1,
12274                        "payload_codec": DEFAULT_CODEC,
12275                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
12276                    }),
12277                    raw: HashMap::new(),
12278                },
12279                HistoryEvent {
12280                    event_type: "SignalReceived".to_string(),
12281                    payload: json!({
12282                        "signal_id": "python-signal-2",
12283                        "signal_name": "increment",
12284                        "workflow_sequence": 2,
12285                        "payload_codec": DEFAULT_CODEC,
12286                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
12287                    }),
12288                    raw: HashMap::new(),
12289                },
12290                HistoryEvent {
12291                    event_type: "SignalReceived".to_string(),
12292                    payload: json!({
12293                        "signal_id": "rust-signal-3",
12294                        "signal_name": "set",
12295                        "workflow_sequence": 3,
12296                        "payload_codec": DEFAULT_CODEC,
12297                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
12298                    }),
12299                    raw: HashMap::new(),
12300                },
12301            ],
12302            history_export: None,
12303            run_status: Some("completed".to_string()),
12304        };
12305
12306        let result = worker.execute_query_task(task).await.expect("query result");
12307        assert_eq!(result.into_json().expect("query projection"), json!(0));
12308    }
12309
12310    #[tokio::test]
12311    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
12312        let worker = replay_counter_worker();
12313        let running_history = json!([
12314            {
12315                "type": "ActivityCompleted",
12316                "payload": {
12317                    "sequence": 1,
12318                    "activity_type": "load-counter",
12319                    "payload_codec": DEFAULT_CODEC,
12320                    "result": fixture_envelope(json!("loaded"))
12321                }
12322            },
12323            {
12324                "type": "SignalWaitOpened",
12325                "payload": {
12326                    "sequence": 3,
12327                    "signal_name": "increment"
12328                }
12329            },
12330            {
12331                "type": "SignalReceived",
12332                "payload": {
12333                    "signal_id": "signal-3",
12334                    "signal_name": "increment",
12335                    "workflow_sequence": 2,
12336                    "payload_codec": DEFAULT_CODEC,
12337                    "arguments": fixture_envelope(json!([3]))
12338                }
12339            },
12340            {
12341                "type": "SignalApplied",
12342                "payload": {
12343                    "sequence": 3,
12344                    "signal_id": "signal-3",
12345                    "signal_name": "increment",
12346                    "payload_codec": DEFAULT_CODEC,
12347                    "value": fixture_envelope(json!([3]))
12348                }
12349            }
12350        ]);
12351
12352        let running = worker
12353            .execute_query_task(replay_counter_query(
12354                "current",
12355                running_history.clone(),
12356                "running",
12357            ))
12358            .await
12359            .expect("running replay query");
12360        assert_eq!(
12361            running.clone().into_json().expect("query projection"),
12362            json!({"loaded": "loaded", "count": 3, "finished": false})
12363        );
12364
12365        let detached = worker
12366            .execute_query_task(replay_counter_query(
12367                "detached-mutation",
12368                running_history.clone(),
12369                "running",
12370            ))
12371            .await
12372            .expect("query mutates only its detached state clone");
12373        assert_eq!(detached.into_json().expect("query projection"), json!(999));
12374        let failed = worker
12375            .execute_query_task(replay_counter_query(
12376                "failed-mutation",
12377                running_history.clone(),
12378                "running",
12379            ))
12380            .await
12381            .expect_err("failed query");
12382        assert_eq!(failed.reason, "query_rejected");
12383        let unchanged = worker
12384            .execute_query_task(replay_counter_query("current", running_history, "running"))
12385            .await
12386            .expect("later query reconstructs unchanged state");
12387        assert_eq!(unchanged, running);
12388
12389        let restarted_worker = replay_counter_worker();
12390        let empty_arguments = fixture_envelope(json!([]));
12391        let loaded_result = fixture_envelope(json!("loaded"));
12392        let signal_three = fixture_blob(json!([3]));
12393        let signal_five = fixture_blob(json!([5]));
12394        let restarted_task: QueryTask = serde_json::from_value(json!({
12395            "query_task_id": "query-after-restart",
12396            "workflow_id": "counter-1",
12397            "run_id": "run-counter-1",
12398            "workflow_type": "replay-counter",
12399            "query_name": "current",
12400            "payload_codec": DEFAULT_CODEC,
12401            "workflow_arguments": empty_arguments.clone(),
12402            "query_arguments": empty_arguments,
12403            "history_events": [],
12404            "history_export": {
12405                "payloads": {"codec": DEFAULT_CODEC},
12406                "history_events": [
12407                    {
12408                        "type": "ActivityCompleted",
12409                        "payload": {
12410                            "sequence": 1,
12411                            "activity_type": "load-counter",
12412                            "payload_codec": DEFAULT_CODEC,
12413                            "result": null
12414                        }
12415                    },
12416                    {
12417                        "type": "SignalWaitOpened",
12418                        "payload": {
12419                            "sequence": 3,
12420                            "signal_name": "increment"
12421                        }
12422                    },
12423                    {
12424                        "type": "SignalReceived",
12425                        "payload": {
12426                            "signal_id": "signal-3",
12427                            "signal_name": "increment",
12428                            "workflow_sequence": 2
12429                        }
12430                    },
12431                    {
12432                        "type": "SignalApplied",
12433                        "payload": {
12434                            "sequence": 3,
12435                            "signal_id": "signal-3",
12436                            "signal_name": "increment"
12437                        }
12438                    },
12439                    {
12440                        "type": "SignalWaitOpened",
12441                        "payload": {
12442                            "sequence": 5,
12443                            "signal_name": "increment"
12444                        }
12445                    },
12446                    {
12447                        "type": "SignalReceived",
12448                        "payload": {
12449                            "signal_id": "signal-5",
12450                            "signal_name": "increment",
12451                            "workflow_sequence": 4
12452                        }
12453                    },
12454                    {
12455                        "type": "SignalApplied",
12456                        "payload": {
12457                            "sequence": 5,
12458                            "signal_id": "signal-5",
12459                            "signal_name": "increment"
12460                        }
12461                    }
12462                ],
12463                "activities": [{
12464                    "sequence": 1,
12465                    "activity_type": "load-counter",
12466                    "payload_codec": DEFAULT_CODEC,
12467                    "result": loaded_result
12468                }],
12469                "signals": [
12470                    {
12471                        "id": "signal-3",
12472                        "name": "increment",
12473                        "workflow_sequence": 2,
12474                        "payload_codec": DEFAULT_CODEC,
12475                        "arguments": signal_three
12476                    },
12477                    {
12478                        "id": "signal-5",
12479                        "name": "increment",
12480                        "workflow_sequence": 4,
12481                        "payload_codec": DEFAULT_CODEC,
12482                        "arguments": signal_five
12483                    }
12484                ]
12485            },
12486            "run_status": "completed"
12487        }))
12488        .expect("cold replay query task");
12489        let completed = restarted_worker
12490            .execute_query_task(restarted_task)
12491            .await
12492            .expect("completed cold replay query");
12493        assert_eq!(
12494            completed.into_json().expect("query projection"),
12495            json!({"loaded": "loaded", "count": 8, "finished": true})
12496        );
12497    }
12498
12499    #[tokio::test]
12500    async fn replayed_query_replay_failures_are_machine_readable() {
12501        let worker = replay_counter_worker();
12502        let task = replay_counter_query(
12503            "current",
12504            json!([{
12505                "type": "ActivityCompleted",
12506                "payload": {
12507                    "sequence": 1,
12508                    "payload_codec": DEFAULT_CODEC,
12509                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
12510                }
12511            }]),
12512            "running",
12513        );
12514        let failure = worker
12515            .execute_query_task(task)
12516            .await
12517            .expect_err("invalid replay history payload");
12518        assert_eq!(failure.reason, "query_payload_decode_failed");
12519        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
12520        assert!(failure.message.contains("invalid_payload_framing"));
12521    }
12522
12523    #[tokio::test]
12524    async fn query_task_restores_compact_history_from_export() {
12525        let client = Client::new("http://127.0.0.1:8080").expect("client");
12526        let mut worker = Worker::new(client, "rust-workers");
12527        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12528        worker.register_query("counter", "current", |ctx, _args| async move {
12529            Ok(json!(ctx.signals("increment")[0][0]))
12530        });
12531        let empty_arguments = fixture_envelope(json!([]));
12532        let exported_signal = fixture_blob(json!([9]));
12533        let task: QueryTask = serde_json::from_value(json!({
12534            "query_task_id": "query-export",
12535            "workflow_type": "counter",
12536            "query_name": "current",
12537            "payload_codec": DEFAULT_CODEC,
12538            "workflow_arguments": empty_arguments.clone(),
12539            "query_arguments": empty_arguments,
12540            "history_events": [],
12541            "history_export": {
12542                "payloads": {"codec": DEFAULT_CODEC},
12543                "history_events": [{
12544                    "type": "SignalReceived",
12545                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
12546                }],
12547                "signals": [{
12548                    "id": "signal-export",
12549                    "name": "increment",
12550                    "status": "applied",
12551                    "workflow_sequence": 1,
12552                    "payload_codec": DEFAULT_CODEC,
12553                    "arguments": exported_signal
12554                }]
12555            }
12556        }))
12557        .expect("query task");
12558
12559        let result = worker.execute_query_task(task).await.expect("query result");
12560        assert_eq!(result.into_json().expect("query projection"), json!(9));
12561    }
12562
12563    #[tokio::test]
12564    async fn query_task_failures_have_stable_reasons() {
12565        let client = Client::new("http://127.0.0.1:8080").expect("client");
12566        let mut worker = Worker::new(client, "rust-workers");
12567        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12568        worker.register_query(
12569            "counter",
12570            "current",
12571            |_ctx, _args| async move { Ok(json!(0)) },
12572        );
12573
12574        let base_task = QueryTask {
12575            query_task_id: "query-errors".to_string(),
12576            query_task_attempt: 1,
12577            lease_owner: None,
12578            workflow_id: Some("counter-errors".to_string()),
12579            run_id: Some("run-errors".to_string()),
12580            workflow_type: "counter".to_string(),
12581            query_name: "missing".to_string(),
12582            payload_codec: DEFAULT_CODEC.to_string(),
12583            workflow_arguments: Some(fixture_envelope(json!([]))),
12584            query_arguments: Some(fixture_envelope(json!([]))),
12585            history_events: Vec::new(),
12586            history_export: None,
12587            run_status: Some("running".to_string()),
12588        };
12589
12590        let unknown = worker
12591            .execute_query_task(base_task.clone())
12592            .await
12593            .expect_err("unknown query");
12594        assert_eq!(unknown.reason, "rejected_unknown_query");
12595
12596        let mut malformed = base_task;
12597        malformed.query_name = "current".to_string();
12598        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
12599        let malformed = worker
12600            .execute_query_task(malformed)
12601            .await
12602            .expect_err("malformed payload");
12603        assert_eq!(malformed.reason, "query_payload_decode_failed");
12604
12605        let client = Client::new("http://127.0.0.1:8080").expect("client");
12606        let mut unavailable_worker = Worker::new(client, "rust-workers");
12607        unavailable_worker
12608            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12609        let empty_arguments = fixture_envelope(json!([]));
12610        let unavailable_task: QueryTask = serde_json::from_value(json!({
12611            "query_task_id": "query-unavailable",
12612            "workflow_type": "counter",
12613            "query_name": "current",
12614            "payload_codec": DEFAULT_CODEC,
12615            "workflow_arguments": empty_arguments.clone(),
12616            "query_arguments": empty_arguments
12617        }))
12618        .expect("query task");
12619        let unavailable = unavailable_worker
12620            .execute_query_task(unavailable_task)
12621            .await
12622            .expect_err("query handler unavailable");
12623        assert_eq!(unavailable.reason, "query_handler_unavailable");
12624    }
12625
12626    #[tokio::test]
12627    async fn client_query_decodes_result_and_typed_failure() {
12628        let server = MockWorkerServer::start();
12629        let client = Client::builder(server.base_url())
12630            .timeout(Duration::from_secs(2))
12631            .build()
12632            .expect("client");
12633
12634        let result = client
12635            .query_workflow("counter-1", "current", json!([]))
12636            .await
12637            .expect("query result");
12638        assert_eq!(result, json!({"count": 8}));
12639
12640        let error = client
12641            .query_workflow("counter-1", "missing", json!([]))
12642            .await
12643            .expect_err("unknown query");
12644        let Error::QueryFailed(failure) = error else {
12645            panic!("expected typed query failure");
12646        };
12647        assert_eq!(failure.status, 404);
12648        assert_eq!(failure.reason, "rejected_unknown_query");
12649    }
12650
12651    #[tokio::test]
12652    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
12653        let server = MockWorkerServer::start();
12654        let client = Client::builder(server.base_url())
12655            .timeout(Duration::from_secs(2))
12656            .build()
12657            .expect("client");
12658        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
12659
12660        client
12661            .start_workflow(
12662                "typed.echo",
12663                "rust-workers",
12664                "typed-start",
12665                arguments.clone(),
12666            )
12667            .await
12668            .expect("typed workflow start");
12669        assert_eq!(
12670            decode_wire_avro_value(
12671                &server.request_body("/api/workflows")["input"],
12672                DEFAULT_CODEC,
12673            )
12674            .expect("typed start input"),
12675            arguments
12676        );
12677
12678        client
12679            .signal_workflow("typed-1", "changed", arguments.clone())
12680            .await
12681            .expect("typed signal");
12682        assert_eq!(
12683            decode_wire_avro_value(
12684                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
12685                DEFAULT_CODEC,
12686            )
12687            .expect("typed signal input"),
12688            arguments
12689        );
12690
12691        assert_eq!(
12692            client
12693                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
12694                .await
12695                .expect("typed query"),
12696            typed_fidelity_probe()
12697        );
12698        assert_eq!(
12699            decode_wire_avro_value(
12700                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
12701                DEFAULT_CODEC,
12702            )
12703            .expect("typed query input"),
12704            arguments
12705        );
12706
12707        assert_eq!(
12708            client
12709                .update_workflow_avro_value(
12710                    "typed-1",
12711                    "replace",
12712                    arguments.clone(),
12713                    Some("typed-request"),
12714                )
12715                .await
12716                .expect("typed update"),
12717            typed_fidelity_probe()
12718        );
12719        let update = server.request_body("/api/workflows/typed-1/update/replace");
12720        assert_eq!(update["request_id"], "typed-request");
12721        assert_eq!(
12722            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
12723            arguments
12724        );
12725
12726        let handle = WorkflowHandle {
12727            client: client.clone(),
12728            workflow_id: "typed-1".to_string(),
12729            run_id: Some("run-typed-1".to_string()),
12730            workflow_type: "typed.echo".to_string(),
12731        };
12732        assert_eq!(
12733            handle
12734                .result_avro_value(WorkflowResultOptions::default())
12735                .await
12736                .expect("typed workflow result"),
12737            typed_fidelity_probe()
12738        );
12739
12740        client
12741            .complete_activity_task(
12742                "activity-typed",
12743                "attempt-typed",
12744                "rust-worker",
12745                typed_fidelity_probe(),
12746                DEFAULT_CODEC,
12747            )
12748            .await
12749            .expect("typed activity completion");
12750        assert_eq!(
12751            decode_wire_avro_value(
12752                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
12753                    ["result"],
12754                DEFAULT_CODEC,
12755            )
12756            .expect("typed activity result"),
12757            typed_fidelity_probe()
12758        );
12759        client
12760            .fail_activity_task(
12761                "activity-typed",
12762                "attempt-typed",
12763                "rust-worker",
12764                "typed failure",
12765                true,
12766            )
12767            .await
12768            .expect("activity failure");
12769    }
12770
12771    #[tokio::test]
12772    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
12773        let server = MockWorkerServer::start();
12774        let client = Client::builder(server.base_url())
12775            .timeout(Duration::from_secs(2))
12776            .build()
12777            .expect("client");
12778
12779        let options = WorkflowCommandOptions::new()
12780            .reason("cleanup requested")
12781            .request_id("cancel-17");
12782        let cancelled = client
12783            .cancel_workflow("wf-lifecycle", options)
12784            .await
12785            .expect("instance cancellation");
12786        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
12787        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
12788        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
12789        assert_eq!(
12790            server.request_body("/api/workflows/wf-lifecycle/cancel"),
12791            json!({"reason":"cleanup requested","request_id":"cancel-17"})
12792        );
12793
12794        let terminated = client
12795            .terminate_workflow(
12796                "wf-lifecycle",
12797                WorkflowCommandOptions::new().reason("forced stop"),
12798            )
12799            .await
12800            .expect("instance termination");
12801        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
12802        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
12803
12804        client
12805            .cancel_workflow_run(
12806                "wf-lifecycle",
12807                "run-current",
12808                WorkflowCommandOptions::default(),
12809            )
12810            .await
12811            .expect("selected run cancellation");
12812        client
12813            .terminate_workflow_run(
12814                "wf-lifecycle",
12815                "run-current",
12816                WorkflowCommandOptions::default(),
12817            )
12818            .await
12819            .expect("selected run termination");
12820
12821        for (command, error) in [
12822            (
12823                WorkflowCommandKind::Cancel,
12824                client
12825                    .cancel_workflow_run(
12826                        "wf-lifecycle",
12827                        "run-stale",
12828                        WorkflowCommandOptions::default(),
12829                    )
12830                    .await
12831                    .expect_err("stale cancellation must be rejected"),
12832            ),
12833            (
12834                WorkflowCommandKind::Terminate,
12835                client
12836                    .terminate_workflow_run(
12837                        "wf-lifecycle",
12838                        "run-stale",
12839                        WorkflowCommandOptions::default(),
12840                    )
12841                    .await
12842                    .expect_err("stale termination must be rejected"),
12843            ),
12844        ] {
12845            let Error::WorkflowCommandRejected(rejection) = error else {
12846                panic!("expected typed command rejection");
12847            };
12848            assert_eq!(rejection.command, command);
12849            assert_eq!(rejection.status, 409);
12850            assert_eq!(rejection.reason, "historical_run_command_rejected");
12851            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12852            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12853        }
12854    }
12855
12856    #[tokio::test]
12857    async fn workflow_start_options_send_server_enforced_deadlines() {
12858        let server = MockWorkerServer::start();
12859        let client = Client::builder(server.base_url())
12860            .timeout(Duration::from_secs(2))
12861            .build()
12862            .expect("client");
12863
12864        let handle = client
12865            .start_workflow_with_options(
12866                "rust.timeout",
12867                "rust-timeouts",
12868                "wf-start-options",
12869                WorkflowStartOptions::new()
12870                    .execution_timeout_seconds(30)
12871                    .run_timeout_seconds(1),
12872                json!([]),
12873            )
12874            .await
12875            .expect("workflow start");
12876
12877        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12878        let body = server.request_body("/api/workflows");
12879        assert_eq!(body["execution_timeout_seconds"], 30);
12880        assert_eq!(body["run_timeout_seconds"], 1);
12881
12882        let invalid = client
12883            .start_workflow_with_options(
12884                "rust.timeout",
12885                "rust-timeouts",
12886                "wf-invalid-options",
12887                WorkflowStartOptions::new()
12888                    .execution_timeout_seconds(1)
12889                    .run_timeout_seconds(2),
12890                json!([]),
12891            )
12892            .await
12893            .expect_err("invalid deadline ordering");
12894        assert!(invalid
12895            .to_string()
12896            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12897    }
12898
12899    #[tokio::test]
12900    async fn workflow_result_returns_each_typed_terminal_outcome() {
12901        let server = MockWorkerServer::start();
12902        let client = Client::builder(server.base_url())
12903            .timeout(Duration::from_secs(2))
12904            .build()
12905            .expect("client");
12906        let options = WorkflowResultOptions {
12907            poll_interval: Duration::ZERO,
12908            timeout: Duration::from_secs(1),
12909        };
12910
12911        let failed = WorkflowHandle {
12912            client: client.clone(),
12913            workflow_id: "wf-failed".to_string(),
12914            run_id: Some("run-failed".to_string()),
12915            workflow_type: "failure".to_string(),
12916        }
12917        .result(options)
12918        .await
12919        .expect_err("failed outcome");
12920        let Error::WorkflowFailed(failure) = failed else {
12921            panic!("expected WorkflowFailed");
12922        };
12923        assert_eq!(failure.workflow_id, "wf-failed");
12924        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12925        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12926        assert_eq!(failure.failure_category.as_deref(), Some("application"));
12927        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12928        assert_eq!(
12929            failure.exception_class.as_deref(),
12930            Some("billing::PaymentError")
12931        );
12932        assert_eq!(failure.non_retryable, Some(true));
12933
12934        for (workflow_id, expected_kind, expected_reason) in [
12935            (
12936                "wf-cancelled",
12937                WorkflowTerminalKind::Cancelled,
12938                "cleanup requested",
12939            ),
12940            (
12941                "wf-terminated",
12942                WorkflowTerminalKind::Terminated,
12943                "forced stop",
12944            ),
12945            (
12946                "wf-timed-out",
12947                WorkflowTerminalKind::TimedOut,
12948                "run_timeout",
12949            ),
12950        ] {
12951            let error = WorkflowHandle {
12952                client: client.clone(),
12953                workflow_id: workflow_id.to_string(),
12954                run_id: None,
12955                workflow_type: "terminal".to_string(),
12956            }
12957            .result(options)
12958            .await
12959            .expect_err("typed terminal outcome");
12960            let outcome = match error {
12961                Error::WorkflowCancelled(outcome) => outcome,
12962                Error::WorkflowTerminated(outcome) => outcome,
12963                Error::WorkflowTimedOut(outcome) => outcome,
12964                other => panic!("unexpected terminal error: {other}"),
12965            };
12966            assert_eq!(outcome.kind, expected_kind);
12967            assert_eq!(outcome.workflow_id, workflow_id);
12968            assert_eq!(outcome.reason, expected_reason);
12969        }
12970
12971        let wait_timeout = WorkflowHandle {
12972            client,
12973            workflow_id: "wf-waiting".to_string(),
12974            run_id: Some("run-waiting".to_string()),
12975            workflow_type: "waiting".to_string(),
12976        }
12977        .result(WorkflowResultOptions {
12978            poll_interval: Duration::ZERO,
12979            timeout: Duration::ZERO,
12980        })
12981        .await
12982        .expect_err("client wait timeout");
12983        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12984            panic!("expected typed client timeout");
12985        };
12986        assert_eq!(timeout.reason, "result_wait_timeout");
12987        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12988        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12989    }
12990
12991    #[tokio::test]
12992    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12993        let server = MockWorkerServer::start();
12994        let client = Client::builder(server.base_url())
12995            .timeout(Duration::from_secs(2))
12996            .build()
12997            .expect("client");
12998
12999        let handle = WorkflowHandle {
13000            client,
13001            workflow_id: "wf-selected".to_string(),
13002            run_id: Some("run-selected".to_string()),
13003            workflow_type: "selected".to_string(),
13004        };
13005        let options = WorkflowResultOptions {
13006            poll_interval: Duration::ZERO,
13007            timeout: Duration::from_secs(1),
13008        };
13009
13010        let current = handle
13011            .result(options)
13012            .await
13013            .expect("instance result follows the current run");
13014        assert_eq!(current, json!("current run output"));
13015
13016        let error = handle
13017            .result_selected_run(options)
13018            .await
13019            .expect_err("the selected run is cancelled even though the current run completed");
13020
13021        let Error::WorkflowCancelled(outcome) = error else {
13022            panic!("expected selected run cancellation");
13023        };
13024        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
13025        assert_eq!(outcome.reason, "selected run cancelled");
13026        assert_eq!(
13027            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
13028            1
13029        );
13030        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
13031    }
13032
13033    #[tokio::test]
13034    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
13035        let server = MockWorkerServer::draining_polls();
13036        let client = Client::builder(server.base_url())
13037            .timeout(Duration::from_secs(2))
13038            .build()
13039            .expect("client");
13040
13041        let workflow = client
13042            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
13043            .await
13044            .expect("workflow drain response");
13045        let activity = client
13046            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
13047            .await
13048            .expect("activity drain response");
13049        let query = client
13050            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
13051            .await
13052            .expect("query drain response");
13053
13054        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
13055            assert_eq!(
13056                outcome,
13057                WorkerPollOutcome::Stop {
13058                    poll_status: Some("draining".to_string()),
13059                    reason: Some("worker_draining".to_string()),
13060                }
13061            );
13062        }
13063
13064        assert!(client
13065            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
13066            .await
13067            .expect("compatibility poll")
13068            .is_none());
13069    }
13070
13071    #[tokio::test]
13072    async fn managed_worker_honors_drain_stop_for_every_task_family() {
13073        let server = MockWorkerServer::draining_polls();
13074        let client = Client::builder(server.base_url())
13075            .timeout(Duration::from_secs(2))
13076            .build()
13077            .expect("client");
13078
13079        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
13080            .worker_id("draining-workflow-worker")
13081            .poll_timeout(Duration::ZERO);
13082        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
13083        workflow_worker
13084            .run()
13085            .await
13086            .expect("workflow drain is a clean stop");
13087
13088        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
13089            .worker_id("draining-activity-worker")
13090            .poll_timeout(Duration::ZERO);
13091        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
13092        activity_worker
13093            .run()
13094            .await
13095            .expect("activity drain is a clean stop");
13096
13097        let mut query_worker = Worker::new(client, "rust-workers")
13098            .worker_id("draining-query-worker")
13099            .poll_timeout(Duration::ZERO);
13100        query_worker.register_query("counter", "current", |_ctx, _args| async {
13101            Ok(Value::Null)
13102        });
13103        query_worker
13104            .run()
13105            .await
13106            .expect("query drain is a clean stop");
13107    }
13108
13109    #[tokio::test]
13110    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
13111        let server = MockWorkerServer::start();
13112        let client = Client::builder(server.base_url())
13113            .timeout(Duration::from_secs(2))
13114            .build()
13115            .expect("client");
13116
13117        let heartbeat = client
13118            .heartbeat_activity_task(
13119                "activity-cancel",
13120                "attempt-cancel",
13121                "rust-worker",
13122                typed_fidelity_probe(),
13123            )
13124            .await
13125            .expect("cancellation heartbeat");
13126        assert!(heartbeat.cancel_requested);
13127        assert!(heartbeat.should_stop());
13128        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
13129        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
13130        let heartbeat_body =
13131            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
13132        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
13133        assert_eq!(
13134            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
13135                .expect("typed heartbeat details"),
13136            typed_fidelity_probe()
13137        );
13138
13139        let error = client
13140            .complete_activity_task(
13141                "activity-cancel",
13142                "attempt-cancel",
13143                "rust-worker",
13144                json!({"late":true}),
13145                DEFAULT_CODEC,
13146            )
13147            .await
13148            .expect_err("late completion must be refused");
13149        assert!(activity_task_rejection_is_final(&error));
13150        let Error::ActivityTaskRejected(rejection) = error else {
13151            panic!("expected typed activity rejection");
13152        };
13153        assert_eq!(rejection.status, 409);
13154        assert_eq!(rejection.reason, "run_cancelled");
13155        assert!(rejection.cancel_requested);
13156        assert_eq!(rejection.can_continue, Some(false));
13157    }
13158
13159    #[tokio::test]
13160    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
13161        let server = MockWorkerServer::cancelled_activity();
13162        let client = Client::builder(server.base_url())
13163            .timeout(Duration::from_secs(2))
13164            .build()
13165            .expect("client");
13166        let cancellation_observed = Arc::new(AtomicBool::new(false));
13167        let observed = Arc::clone(&cancellation_observed);
13168        let mut worker = Worker::new(client.clone(), "rust-workers")
13169            .worker_id("rust-cancel-worker")
13170            .poll_timeout(Duration::from_millis(10));
13171        worker.register_activity("cancel-aware", move |ctx, _args| {
13172            let observed = Arc::clone(&observed);
13173            async move {
13174                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
13175                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
13176                Ok(json!({"late":"completion"}))
13177            }
13178        });
13179
13180        assert_eq!(
13181            worker.run_once().await.expect("cancelled attempt handled"),
13182            1
13183        );
13184        assert!(cancellation_observed.load(Ordering::SeqCst));
13185        assert_eq!(
13186            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
13187            1
13188        );
13189
13190        let mut restarted = Worker::new(client, "rust-workers")
13191            .worker_id("rust-cancel-worker-restarted")
13192            .poll_timeout(Duration::from_millis(10));
13193        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
13194        assert_eq!(
13195            restarted
13196                .run_once()
13197                .await
13198                .expect("replacement worker continues polling"),
13199            0
13200        );
13201    }
13202
13203    #[tokio::test]
13204    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
13205        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
13206        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
13207        let client = Client::builder(server.base_url())
13208            .timeout(Duration::from_secs(2))
13209            .build()
13210            .expect("client");
13211
13212        let direct_error = client
13213            .complete_workflow_task(
13214                "workflow-timeout-task",
13215                "timeout-worker",
13216                3,
13217                vec![json!({
13218                    "type": "complete_workflow",
13219                    "result": fixture_envelope(Value::Null)
13220                })],
13221            )
13222            .await
13223            .expect_err("the low-level client preserves the completion rejection");
13224        let Error::Http { status, body } = direct_error else {
13225            panic!("expected the original HTTP completion rejection");
13226        };
13227        assert_eq!(status, reqwest::StatusCode::CONFLICT);
13228        assert_eq!(
13229            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
13230            "run_timed_out"
13231        );
13232
13233        let mut worker = Worker::new(client, "rust-workers")
13234            .worker_id("timeout-worker")
13235            .poll_timeout(Duration::from_millis(10));
13236        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
13237            Ok(json!({"late": "result"}))
13238        });
13239
13240        assert_eq!(
13241            worker
13242                .run_once()
13243                .await
13244                .expect("authoritative selected-run timeout settles the tick"),
13245            1
13246        );
13247        assert_eq!(
13248            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
13249            2,
13250            "both the direct client proof and managed worker must see the rejection"
13251        );
13252    }
13253
13254    #[tokio::test]
13255    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
13256        for (name, status, response) in [
13257            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
13258            (
13259                "command was recorded",
13260                "409 Conflict",
13261                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13262            ),
13263            (
13264                "lease conflict",
13265                "409 Conflict",
13266                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
13267            ),
13268            (
13269                "nonterminal run",
13270                "409 Conflict",
13271                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
13272            ),
13273            (
13274                "different selected run",
13275                "409 Conflict",
13276                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
13277            ),
13278            (
13279                "different task attempt",
13280                "409 Conflict",
13281                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13282            ),
13283            (
13284                "authentication failure",
13285                "401 Unauthorized",
13286                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13287            ),
13288            (
13289                "authorization failure",
13290                "403 Forbidden",
13291                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13292            ),
13293            (
13294                "protocol failure",
13295                "400 Bad Request",
13296                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
13297            ),
13298            (
13299                "malformed command",
13300                "422 Unprocessable Entity",
13301                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13302            ),
13303            (
13304                "transient server failure",
13305                "503 Service Unavailable",
13306                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
13307            ),
13308        ] {
13309            let server = MockWorkerServer::workflow_completion(status, response);
13310            let client = Client::builder(server.base_url())
13311                .timeout(Duration::from_secs(2))
13312                .build()
13313                .expect("client");
13314            let mut worker = Worker::new(client, "rust-workers")
13315                .worker_id("timeout-worker")
13316                .poll_timeout(Duration::from_millis(10));
13317            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
13318                Ok(json!({"late": "result"}))
13319            });
13320
13321            let error = worker
13322                .run_once()
13323                .await
13324                .expect_err(&format!("{name} must remain an error"));
13325            assert!(
13326                matches!(error, Error::Http { .. } | Error::Protocol(_)),
13327                "{name} returned an unexpected error variant: {error}"
13328            );
13329        }
13330    }
13331
13332    #[tokio::test]
13333    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
13334        let server = MockWorkerServer::start();
13335        let client = Client::builder(server.base_url())
13336            .worker_token(Some("worker-secret".to_string()))
13337            .namespace("orders")
13338            .timeout(Duration::from_secs(2))
13339            .build()
13340            .expect("client");
13341        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
13342
13343        let result = client
13344            .deregister_worker_registration("worker/α space")
13345            .await
13346            .expect("deregister worker registration");
13347
13348        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
13349        assert_eq!(
13350            server.worker_protocol_for(path).as_deref(),
13351            Some(WORKER_PROTOCOL_VERSION)
13352        );
13353        assert_eq!(server.control_protocol_for(path), None);
13354        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
13355        assert_eq!(
13356            server.authorization_for(path).as_deref(),
13357            Some("Bearer worker-secret")
13358        );
13359        assert_eq!(
13360            result,
13361            WorkerDeregistrationEnvelope {
13362                worker_id: "deregistered-worker".to_string(),
13363                outcome: "deregistered".to_string(),
13364                recovered_workflow_task_count: 2,
13365            }
13366        );
13367    }
13368
13369    #[tokio::test]
13370    async fn low_level_registration_rejects_update_validators_before_transport() {
13371        let server = MockWorkerServer::start();
13372        let client = Client::builder(server.base_url())
13373            .timeout(Duration::from_secs(2))
13374            .build()
13375            .expect("client");
13376
13377        for update_validators in [json!(["approve"]), json!("approve")] {
13378            let error = client
13379                .register_worker_with_command_contracts(
13380                    "validator-claiming-worker",
13381                    "rust-workers",
13382                    vec!["orders".to_string()],
13383                    vec![],
13384                    1,
13385                    1,
13386                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
13387                    json!({
13388                        "orders": {
13389                            "queries": ["current"],
13390                            "updates": ["approve"],
13391                            "update_validators": update_validators,
13392                        },
13393                    }),
13394                )
13395                .await
13396                .expect_err("unsupported validator claims must fail before registration");
13397
13398            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
13399                panic!("expected typed unsupported-validator failure");
13400            };
13401            assert_eq!(workflow_type, "orders");
13402        }
13403        assert_eq!(server.request_count("/api/worker/register"), 0);
13404    }
13405
13406    #[tokio::test]
13407    async fn low_level_registration_preserves_query_and_update_contracts() {
13408        let server = MockWorkerServer::start();
13409        let client = Client::builder(server.base_url())
13410            .timeout(Duration::from_secs(2))
13411            .build()
13412            .expect("client");
13413        let contracts = json!({
13414            "orders": {
13415                "queries": ["current"],
13416                "updates": ["approve"],
13417                "update_validators": [],
13418            },
13419            "payments": {
13420                "queries": ["status"],
13421                "updates": ["capture"],
13422            },
13423        });
13424
13425        client
13426            .register_worker_with_command_contracts(
13427                "command-worker",
13428                "rust-workers",
13429                vec!["orders".to_string(), "payments".to_string()],
13430                vec![],
13431                1,
13432                1,
13433                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
13434                contracts.clone(),
13435            )
13436            .await
13437            .expect("query and update contracts must remain supported");
13438
13439        assert_eq!(
13440            server.request_body("/api/worker/register")["workflow_command_contracts"],
13441            contracts
13442        );
13443    }
13444
13445    #[tokio::test]
13446    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
13447        let server = MockWorkerServer::start();
13448        let control_only = Client::builder(server.base_url())
13449            .control_token(Some("control-secret".to_string()))
13450            .build()
13451            .expect("control client");
13452
13453        let error = control_only
13454            .register_worker("worker", "queue", vec![], vec![], 1, 1)
13455            .await
13456            .expect_err("control token must not authorize a worker request");
13457        assert!(matches!(
13458            error,
13459            Error::MissingRoleCredentials { role: "worker", .. }
13460        ));
13461        assert_eq!(server.request_count("/api/worker/register"), 0);
13462
13463        let worker_only = Client::builder(server.base_url())
13464            .worker_token(Some("worker-secret".to_string()))
13465            .build()
13466            .expect("worker client");
13467        let error = worker_only
13468            .health()
13469            .await
13470            .expect_err("worker token must not authorize a control request");
13471        assert!(matches!(
13472            error,
13473            Error::MissingRoleCredentials {
13474                role: "control",
13475                ..
13476            }
13477        ));
13478        assert_eq!(server.request_count("/api/health"), 0);
13479    }
13480
13481    #[tokio::test]
13482    async fn shared_token_supports_worker_and_control_planes() {
13483        let server = MockWorkerServer::start();
13484        let client = Client::builder(server.base_url())
13485            .token(Some("shared-secret".to_string()))
13486            .build()
13487            .expect("client");
13488
13489        client.health().await.expect("control request");
13490        client
13491            .register_worker("worker", "queue", vec![], vec![], 1, 1)
13492            .await
13493            .expect("worker request");
13494
13495        assert_eq!(
13496            server.authorization_for("/api/health").as_deref(),
13497            Some("Bearer shared-secret")
13498        );
13499        assert_eq!(
13500            server.control_protocol_for("/api/health").as_deref(),
13501            Some(CONTROL_PLANE_VERSION)
13502        );
13503        assert_eq!(
13504            server.authorization_for("/api/worker/register").as_deref(),
13505            Some("Bearer shared-secret")
13506        );
13507        assert_eq!(
13508            server
13509                .worker_protocol_for("/api/worker/register")
13510                .as_deref(),
13511            Some(WORKER_PROTOCOL_VERSION)
13512        );
13513    }
13514
13515    #[tokio::test]
13516    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
13517        let server = MockWorkerServer::start();
13518        let client = Client::builder(server.base_url())
13519            .timeout(Duration::from_secs(2))
13520            .build()
13521            .expect("client");
13522
13523        client
13524            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
13525            .await
13526            .expect("register");
13527        client
13528            .heartbeat_worker("capture-worker", 1, 1)
13529            .await
13530            .expect("heartbeat");
13531        client
13532            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
13533            .await
13534            .expect("workflow poll");
13535        client
13536            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
13537            .await
13538            .expect("activity poll");
13539
13540        for path in [
13541            "/api/worker/register",
13542            "/api/worker/heartbeat",
13543            "/api/worker/workflow-tasks/poll",
13544            "/api/worker/activity-tasks/poll",
13545        ] {
13546            assert_eq!(
13547                server.worker_protocol_for(path).as_deref(),
13548                Some(WORKER_PROTOCOL_VERSION),
13549                "unexpected protocol for {path}"
13550            );
13551        }
13552
13553        assert_eq!(
13554            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
13555            1
13556        );
13557        assert_eq!(
13558            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
13559            1
13560        );
13561        assert!(
13562            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
13563                .as_str()
13564                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
13565        );
13566        assert!(
13567            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
13568                .as_str()
13569                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
13570        );
13571    }
13572
13573    #[tokio::test]
13574    async fn query_task_endpoints_send_the_query_feature_protocol() {
13575        let server = MockWorkerServer::start();
13576        let client = Client::builder(server.base_url())
13577            .timeout(Duration::from_secs(2))
13578            .build()
13579            .expect("client");
13580
13581        client
13582            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13583            .await
13584            .expect("query poll");
13585        client
13586            .complete_query_task(
13587                "query-capture",
13588                "capture-worker",
13589                1,
13590                json!(8),
13591                DEFAULT_CODEC,
13592            )
13593            .await
13594            .expect("query complete");
13595        client
13596            .fail_query_task(
13597                "query-capture",
13598                "capture-worker",
13599                1,
13600                "failed",
13601                "query_rejected",
13602                "QueryFailed",
13603            )
13604            .await
13605            .expect("query fail");
13606
13607        for path in [
13608            "/api/worker/query-tasks/poll",
13609            "/api/worker/query-tasks/query-capture/complete",
13610            "/api/worker/query-tasks/query-capture/fail",
13611        ] {
13612            assert_eq!(
13613                server.worker_protocol_for(path).as_deref(),
13614                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
13615                "unexpected protocol for {path}"
13616            );
13617        }
13618
13619        assert_eq!(
13620            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
13621            1
13622        );
13623        assert!(
13624            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
13625                .as_str()
13626                .is_some_and(|id| id.starts_with("rust-query-poll-"))
13627        );
13628    }
13629
13630    #[tokio::test]
13631    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
13632        let server = MockWorkerServer::transient_worker_failures();
13633        let client = Client::builder(server.base_url())
13634            .timeout(Duration::from_secs(2))
13635            .build()
13636            .expect("client");
13637
13638        client
13639            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
13640            .await
13641            .expect("workflow poll retry");
13642        client
13643            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
13644            .await
13645            .expect("activity poll retry");
13646        client
13647            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13648            .await
13649            .expect("query poll retry");
13650
13651        for path in [
13652            "/api/worker/workflow-tasks/poll",
13653            "/api/worker/activity-tasks/poll",
13654            "/api/worker/query-tasks/poll",
13655        ] {
13656            let bodies = server.request_bodies(path);
13657            assert_eq!(bodies.len(), 2, "{path} must be retried once");
13658            assert_eq!(
13659                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
13660                "{path} must preserve the request binding across retry"
13661            );
13662        }
13663    }
13664
13665    #[tokio::test]
13666    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
13667        let server = MockWorkerServer::consecutive_poll_failures(2);
13668        let client = Client::builder(server.base_url())
13669            .timeout(Duration::from_secs(2))
13670            .build()
13671            .expect("client");
13672        let mut worker = Worker::new(client, "capture")
13673            .worker_id("capture-worker")
13674            .poll_timeout(Duration::from_millis(10))
13675            .retry_policy(WorkerRetryPolicy {
13676                max_retries: 2,
13677                initial_backoff: Duration::from_millis(1),
13678                max_backoff: Duration::from_millis(1),
13679            });
13680        worker.register_workflow(
13681            "capture.workflow",
13682            |_ctx, _input| async move { Ok(Value::Null) },
13683        );
13684        worker.register_activity(
13685            "capture.activity",
13686            |_ctx, _input| async move { Ok(Value::Null) },
13687        );
13688        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
13689            Ok(Value::Null)
13690        });
13691
13692        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
13693
13694        for path in [
13695            "/api/worker/workflow-tasks/poll",
13696            "/api/worker/activity-tasks/poll",
13697            "/api/worker/query-tasks/poll",
13698        ] {
13699            let bodies = server.request_bodies(path);
13700            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
13701            assert!(
13702                bodies
13703                    .iter()
13704                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
13705                "{path} must preserve one request binding across every retry"
13706            );
13707        }
13708    }
13709
13710    #[tokio::test]
13711    async fn query_protocol_rejection_from_older_server_is_typed() {
13712        let server = MockWorkerServer::reject_query_protocol();
13713        let client = Client::builder(server.base_url())
13714            .timeout(Duration::from_secs(2))
13715            .build()
13716            .expect("client");
13717
13718        let error = client
13719            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
13720            .await
13721            .expect_err("server below query protocol floor must reject");
13722        let Error::Protocol(failure) = error else {
13723            panic!("expected typed protocol failure");
13724        };
13725
13726        assert_eq!(failure.status, 400);
13727        assert_eq!(failure.reason, "unsupported_protocol_version");
13728        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
13729        assert_eq!(
13730            failure.requested_version.as_deref(),
13731            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
13732        );
13733        assert_eq!(
13734            server
13735                .worker_protocol_for("/api/worker/query-tasks/poll")
13736                .as_deref(),
13737            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
13738        );
13739    }
13740
13741    #[tokio::test]
13742    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
13743        let server = MockWorkerServer::reject_query_protocol();
13744        let client = Client::builder(server.base_url())
13745            .timeout(Duration::from_secs(2))
13746            .build()
13747            .expect("client");
13748        let mut worker = Worker::new(client, "rust-workers")
13749            .worker_id("baseline-worker")
13750            .poll_timeout(Duration::from_millis(10));
13751
13752        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
13753            Ok(Value::Null)
13754        });
13755
13756        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
13757        assert_eq!(
13758            server
13759                .worker_protocol_for("/api/worker/workflow-tasks/poll")
13760                .as_deref(),
13761            Some(WORKER_PROTOCOL_VERSION)
13762        );
13763        assert_eq!(
13764            server.worker_protocol_for("/api/worker/query-tasks/poll"),
13765            None,
13766            "a worker without query handlers must not use the query-task endpoint"
13767        );
13768    }
13769
13770    #[tokio::test]
13771    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
13772        let server = MockWorkerServer::reject_query_completion();
13773        let client = Client::builder(server.base_url())
13774            .timeout(Duration::from_secs(2))
13775            .build()
13776            .expect("client");
13777
13778        let error = client
13779            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
13780            .await
13781            .expect_err("expired completion must be rejected");
13782        let Error::QueryFailed(failure) = error else {
13783            panic!("expected typed query failure");
13784        };
13785        assert_eq!(failure.status, 409);
13786        assert_eq!(failure.reason, "query_task_timed_out");
13787
13788        let mut worker = Worker::new(client, "rust-workers")
13789            .worker_id("late-worker")
13790            .poll_timeout(Duration::from_millis(10));
13791        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13792        worker.register_query(
13793            "counter",
13794            "current",
13795            |_ctx, _args| async move { Ok(json!(8)) },
13796        );
13797
13798        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
13799        assert_eq!(
13800            worker
13801                .run_once()
13802                .await
13803                .expect("worker continues after late completion"),
13804            0
13805        );
13806        assert_eq!(
13807            server.request_count("/api/worker/query-tasks/query-late/complete"),
13808            2
13809        );
13810        assert_eq!(
13811            server.request_count("/api/worker/query-tasks/query-late/fail"),
13812            0,
13813            "a server completion rejection must not be reported as an encoding failure"
13814        );
13815    }
13816
13817    #[tokio::test]
13818    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
13819        let server = MockWorkerServer::start();
13820        let client = Client::builder(server.base_url())
13821            .timeout(Duration::from_secs(2))
13822            .build()
13823            .expect("client");
13824        let mut worker = Worker::new(client, "rust-workers")
13825            .worker_id("joined-worker")
13826            .poll_timeout(Duration::from_millis(10));
13827        worker.register_workflow(
13828            "joined.workflow",
13829            |_ctx, _input| async move { Ok(Value::Null) },
13830        );
13831        worker.register_activity(
13832            "joined.activity",
13833            |_ctx, _input| async move { Ok(Value::Null) },
13834        );
13835        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
13836            Ok(Value::Null)
13837        });
13838
13839        worker
13840            .run_until(tokio::time::sleep(Duration::from_millis(20)))
13841            .await
13842            .expect("normal shutdown");
13843
13844        let deregistration_path = "/api/worker/registrations/mock-worker";
13845        assert_eq!(server.request_count(deregistration_path), 1);
13846        for poll_path in [
13847            "/api/worker/workflow-tasks/poll",
13848            "/api/worker/activity-tasks/poll",
13849            "/api/worker/query-tasks/poll",
13850        ] {
13851            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
13852        }
13853        assert_eq!(
13854            server.captured_paths().last().map(String::as_str),
13855            Some(deregistration_path),
13856            "deregistration must start only after every poller has joined"
13857        );
13858    }
13859
13860    #[tokio::test]
13861    async fn registration_failure_does_not_deregister() {
13862        let server = MockWorkerServer::rejected_registration();
13863        let client = Client::builder(server.base_url())
13864            .timeout(Duration::from_secs(2))
13865            .build()
13866            .expect("client");
13867        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
13868
13869        let error = worker
13870            .run_until(async {})
13871            .await
13872            .expect_err("registration must fail");
13873        assert!(matches!(
13874            error,
13875            Error::Http {
13876                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
13877                ..
13878            }
13879        ));
13880        assert!(server
13881            .captured_paths()
13882            .iter()
13883            .all(|path| !path.starts_with("/api/worker/registrations/")));
13884    }
13885
13886    #[tokio::test]
13887    async fn declined_registration_does_not_deregister() {
13888        let server = MockWorkerServer::declined_registration();
13889        let client = Client::builder(server.base_url())
13890            .timeout(Duration::from_secs(2))
13891            .build()
13892            .expect("client");
13893        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
13894
13895        let error = worker
13896            .run_until(async {})
13897            .await
13898            .expect_err("declined registration must fail");
13899        assert!(matches!(error, Error::WorkerLoop(_)));
13900        assert!(error.to_string().contains("was not accepted"));
13901        assert!(server
13902            .captured_paths()
13903            .iter()
13904            .all(|path| !path.starts_with("/api/worker/registrations/")));
13905    }
13906
13907    #[tokio::test]
13908    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
13909        let server = MockWorkerServer::rejected_deregistration();
13910        let client = Client::builder(server.base_url())
13911            .timeout(Duration::from_secs(2))
13912            .build()
13913            .expect("client");
13914        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
13915
13916        let error = worker
13917            .run_until(async {})
13918            .await
13919            .expect_err("deregistration must fail");
13920        assert!(matches!(
13921            error,
13922            Error::Http {
13923                status: reqwest::StatusCode::FORBIDDEN,
13924                ..
13925            }
13926        ));
13927        assert_eq!(
13928            server.request_count("/api/worker/registrations/mock-worker"),
13929            1
13930        );
13931    }
13932
13933    #[tokio::test]
13934    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13935        let server = MockWorkerServer::rejected_deregistration_protocol();
13936        let client = Client::builder(server.base_url())
13937            .timeout(Duration::from_secs(2))
13938            .build()
13939            .expect("client");
13940        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13941
13942        let error = worker
13943            .run_until(async {})
13944            .await
13945            .expect_err("protocol rejection must fail shutdown");
13946        let Error::Protocol(failure) = error else {
13947            panic!("expected typed protocol failure");
13948        };
13949        assert_eq!(failure.reason, "unsupported_protocol_version");
13950        assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13951        assert_eq!(
13952            server.request_count("/api/worker/registrations/mock-worker"),
13953            1
13954        );
13955    }
13956
13957    #[tokio::test]
13958    async fn primary_poller_error_retains_deregistration_failure_context() {
13959        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13960        let client = Client::builder(server.base_url())
13961            .timeout(Duration::from_secs(2))
13962            .build()
13963            .expect("client");
13964        let mut worker = Worker::new(client, "rust-workers")
13965            .worker_id("combined-failure")
13966            .poll_timeout(Duration::from_millis(10));
13967        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13968            Ok(Value::Null)
13969        });
13970
13971        let error = worker
13972            .run()
13973            .await
13974            .expect_err("worker and cleanup must fail");
13975        let summary = error.to_string();
13976        assert!(summary.contains("authentication_failed"));
13977        assert!(summary.contains("worker cannot deregister"));
13978        let Error::WorkerShutdown {
13979            primary,
13980            deregistration,
13981        } = error
13982        else {
13983            panic!("expected combined worker shutdown error");
13984        };
13985        assert!(matches!(
13986            *primary,
13987            Error::Http {
13988                status: reqwest::StatusCode::UNAUTHORIZED,
13989                ..
13990            }
13991        ));
13992        assert!(matches!(
13993            *deregistration,
13994            Error::Http {
13995                status: reqwest::StatusCode::FORBIDDEN,
13996                ..
13997            }
13998        ));
13999        assert_eq!(
14000            server.request_count("/api/worker/registrations/mock-worker"),
14001            1
14002        );
14003    }
14004
14005    #[tokio::test]
14006    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
14007        let server = MockWorkerServer::start();
14008        let client = Client::builder(server.base_url())
14009            .timeout(Duration::from_secs(2))
14010            .build()
14011            .expect("client");
14012        let mut worker = Worker::new(client, "rust-workers")
14013            .worker_id("activity-only-worker")
14014            .poll_timeout(Duration::from_millis(10));
14015
14016        worker.register_activity(
14017            "activity.only",
14018            |_ctx, _args| async move { Ok(Value::Null) },
14019        );
14020
14021        worker.run_until(async {}).await.expect("run worker");
14022    }
14023
14024    #[tokio::test]
14025    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
14026        let server = MockWorkerServer::start();
14027        let client = Client::builder(server.base_url())
14028            .timeout(Duration::from_secs(2))
14029            .build()
14030            .expect("client");
14031        let mut worker = Worker::new(client, "rust-workers")
14032            .worker_id("workflow-only-worker")
14033            .poll_timeout(Duration::from_millis(10));
14034
14035        worker.register_workflow(
14036            "workflow.only",
14037            |_ctx, _input| async move { Ok(Value::Null) },
14038        );
14039
14040        worker.run_until(async {}).await.expect("run worker");
14041    }
14042
14043    #[tokio::test]
14044    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
14045        let server = MockWorkerServer::start();
14046        let client = Client::builder(server.base_url())
14047            .timeout(Duration::from_secs(2))
14048            .build()
14049            .expect("client");
14050        let observations = Arc::new(Mutex::new(Vec::new()));
14051        let observed = Arc::clone(&observations);
14052        let mut worker = Worker::new(client, "rust-workers")
14053            .worker_id("observed-heartbeat-worker")
14054            .poll_timeout(Duration::from_millis(10))
14055            .on_worker_heartbeat(move |observation| {
14056                observed
14057                    .lock()
14058                    .expect("heartbeat observations")
14059                    .push(observation.clone());
14060            });
14061
14062        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
14063            Ok(Value::Null)
14064        });
14065        let acknowledged = Arc::clone(&observations);
14066        worker
14067            .run_until(async move {
14068                tokio::time::timeout(Duration::from_secs(2), async move {
14069                    loop {
14070                        if !acknowledged
14071                            .lock()
14072                            .expect("heartbeat observations")
14073                            .is_empty()
14074                        {
14075                            break;
14076                        }
14077                        tokio::time::sleep(Duration::from_millis(1)).await;
14078                    }
14079                })
14080                .await
14081                .expect("heartbeat acknowledgement within timeout");
14082            })
14083            .await
14084            .expect("run worker");
14085
14086        let observations = observations.lock().expect("heartbeat observations");
14087        let first = observations.first().expect("heartbeat acknowledgement");
14088        assert_eq!(first.worker_id, "observed-heartbeat-worker");
14089        assert_eq!(first.task_queue, "rust-workers");
14090        assert!(first.acknowledged_at_unix_millis > 0);
14091        assert_eq!(first.acknowledgement, json!({}));
14092    }
14093
14094    #[tokio::test]
14095    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
14096        let server = MockWorkerServer::delayed_heartbeat_worker();
14097        let client = Client::builder(server.base_url())
14098            .timeout(Duration::from_secs(3))
14099            .build()
14100            .expect("client");
14101        let observations = Arc::new(Mutex::new(Vec::new()));
14102        let observed = Arc::clone(&observations);
14103        let mut worker = Worker::new(client, "rust-snapshot-workers")
14104            .worker_id("rust-snapshot-worker")
14105            .poll_timeout(Duration::from_millis(10))
14106            .on_worker_heartbeat(move |observation| {
14107                observed
14108                    .lock()
14109                    .expect("heartbeat observations")
14110                    .push(observation.clone());
14111            });
14112
14113        worker.register_workflow("snapshot", |ctx, _input| async move {
14114            ctx.wait_signal("finish").await?;
14115            Ok(json!({"status": "finished"}))
14116        });
14117        worker.register_query("snapshot", "current", |ctx, _args| async move {
14118            Ok(json!(ctx
14119                .signals("increment")
14120                .iter()
14121                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
14122                .sum::<i64>()))
14123        });
14124        worker.register_activity("cancel-aware", |_ctx, _args| async move {
14125            Ok(json!({"late": "completion"}))
14126        });
14127
14128        worker
14129            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
14130            .await
14131            .expect("delayed heartbeat must allow a clean worker shutdown");
14132
14133        let observations = observations.lock().expect("heartbeat observations");
14134        assert!(
14135            observations.len() >= 3,
14136            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
14137        );
14138        assert!(
14139            observations.windows(2).all(|pair| {
14140                pair[1].acknowledged_at_unix_millis
14141                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
14142                    >= 850
14143            }),
14144            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
14145        );
14146        drop(observations);
14147
14148        let heartbeat_times = server.request_times("/api/worker/heartbeat");
14149        let delayed_request_at = *heartbeat_times
14150            .get(1)
14151            .expect("intentionally delayed heartbeat request");
14152        let delay_window_start = delayed_request_at + Duration::from_millis(100);
14153        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
14154        for path in [
14155            "/api/worker/workflow-tasks/poll",
14156            "/api/worker/activity-tasks/poll",
14157            "/api/worker/query-tasks/poll",
14158        ] {
14159            assert!(
14160                server
14161                    .request_times(path)
14162                    .iter()
14163                    .any(|received_at| *received_at >= delay_window_start
14164                        && *received_at <= delay_window_end),
14165                "{path} must keep polling while a heartbeat acknowledgement is delayed"
14166            );
14167        }
14168        assert!(
14169            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
14170            "workflow work must be settled"
14171        );
14172        assert!(
14173            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
14174            "activity work must be settled"
14175        );
14176        assert!(
14177            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
14178            "query work must be settled"
14179        );
14180    }
14181
14182    #[tokio::test]
14183    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
14184        let server = MockWorkerServer::heartbeat_retry_worker();
14185        let client = Client::builder(server.base_url())
14186            .timeout(Duration::from_secs(2))
14187            .build()
14188            .expect("client");
14189        let observations = Arc::new(Mutex::new(Vec::new()));
14190        let observed = Arc::clone(&observations);
14191        let worker = Worker::new(client, "rust-workers")
14192            .worker_id("heartbeat-retry-worker")
14193            .retry_policy(WorkerRetryPolicy {
14194                max_retries: 1,
14195                initial_backoff: Duration::from_millis(300),
14196                max_backoff: Duration::from_millis(300),
14197            })
14198            .on_worker_heartbeat(move |observation| {
14199                observed
14200                    .lock()
14201                    .expect("heartbeat observations")
14202                    .push(observation.clone());
14203            });
14204
14205        worker
14206            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
14207            .await
14208            .expect("retryable heartbeat failure must remain bounded and recover");
14209
14210        let observations = observations.lock().expect("heartbeat observations");
14211        assert!(observations.len() >= 3, "heartbeat retry must recover");
14212        assert!(
14213            observations.windows(2).all(|pair| {
14214                pair[1]
14215                    .acknowledged_at_unix_millis
14216                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
14217                    >= 850
14218            }),
14219            "a successful retry must start a fresh advertised cadence: {observations:?}"
14220        );
14221        assert_eq!(
14222            server.request_count("/api/worker/heartbeat"),
14223            observations.len() + 1,
14224            "one retryable failure must add exactly one bounded request"
14225        );
14226    }
14227
14228    #[tokio::test]
14229    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
14230        let server = MockWorkerServer::waiting_query_worker();
14231        let client = Client::builder(server.base_url())
14232            .timeout(Duration::from_secs(2))
14233            .build()
14234            .expect("client");
14235        let observations = Arc::new(Mutex::new(Vec::new()));
14236        let observed = Arc::clone(&observations);
14237        let mut worker = Worker::new(client, "rust-snapshot-workers")
14238            .worker_id("rust-snapshot-worker")
14239            .poll_timeout(Duration::from_millis(10))
14240            .on_worker_heartbeat(move |observation| {
14241                observed
14242                    .lock()
14243                    .expect("heartbeat observations")
14244                    .push(observation.clone());
14245            });
14246
14247        worker.register_workflow("snapshot", |ctx, _input| async move {
14248            ctx.wait_signal("finish").await?;
14249            Ok(json!({"status": "finished"}))
14250        });
14251        worker.register_query("snapshot", "current", |ctx, _args| async move {
14252            let current = ctx
14253                .signals("increment")
14254                .iter()
14255                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
14256                .sum::<i64>();
14257            Ok(json!(current))
14258        });
14259        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
14260
14261        worker
14262            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
14263            .await
14264            .expect("pending workflow and query poller must remain live until shutdown");
14265
14266        assert!(
14267            observations.lock().expect("heartbeat observations").len() >= 4,
14268            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
14269        );
14270        assert!(
14271            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
14272            "workflow polling must continue after empty replay acknowledgements"
14273        );
14274        assert!(
14275            server.request_count("/api/worker/query-tasks/poll") >= 2,
14276            "query polling must continue after serving the current query"
14277        );
14278        assert_eq!(
14279            server.request_body("/api/worker/register")["capabilities"],
14280            json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
14281        );
14282        assert_eq!(
14283            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
14284            json!({
14285                "queries": ["current"],
14286                "updates": ["replace"],
14287                "update_validators": [],
14288            })
14289        );
14290
14291        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
14292        assert_eq!(
14293            opened["commands"],
14294            json!([{
14295                "type": "open_signal_wait",
14296                "signal_name": "finish",
14297            }])
14298        );
14299
14300        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
14301            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
14302            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
14303            let failure = server.request_body(&fail_path);
14304            assert_eq!(
14305                failure["failure"]["type"],
14306                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
14307            );
14308            assert_eq!(server.request_count(&completion_path), 0);
14309        }
14310
14311        let query_completion =
14312            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
14313        assert_eq!(query_completion["result"], json!(8));
14314
14315        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
14316        assert_eq!(
14317            server.request_count(terminal_path),
14318            1,
14319            "the matching signal must settle the workflow exactly once"
14320        );
14321        let terminal = server.request_body(terminal_path);
14322        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
14323        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
14324        assert_eq!(
14325            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
14326                .expect("terminal workflow result"),
14327            json!({"status": "finished"})
14328        );
14329    }
14330
14331    #[tokio::test]
14332    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
14333        let server = MockWorkerServer::transient_worker_failures();
14334        let client = Client::builder(server.base_url())
14335            .timeout(Duration::from_secs(2))
14336            .build()
14337            .expect("client");
14338        let mut worker = Worker::new(client, "rust-workers")
14339            .worker_id("retry-worker")
14340            .poll_timeout(Duration::from_millis(10))
14341            .retry_policy(WorkerRetryPolicy {
14342                max_retries: 2,
14343                initial_backoff: Duration::from_millis(1),
14344                max_backoff: Duration::from_millis(1),
14345            });
14346        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14347        worker.register_activity(
14348            "counter.activity",
14349            |_ctx, _input| async move { Ok(Value::Null) },
14350        );
14351        worker.register_query(
14352            "counter",
14353            "current",
14354            |_ctx, _args| async move { Ok(json!(8)) },
14355        );
14356
14357        worker
14358            .run_until(tokio::time::sleep(Duration::from_millis(75)))
14359            .await
14360            .expect("transient failures must not stop the worker");
14361
14362        for path in [
14363            "/api/worker/heartbeat",
14364            "/api/worker/workflow-tasks/poll",
14365            "/api/worker/activity-tasks/poll",
14366            "/api/worker/query-tasks/poll",
14367        ] {
14368            assert!(
14369                server.request_count(path) >= 2,
14370                "{path} must continue after its transient failure"
14371            );
14372        }
14373    }
14374
14375    #[tokio::test]
14376    async fn worker_bounds_transport_retries() {
14377        let server = MockWorkerServer::unavailable_polls();
14378        let client = Client::builder(server.base_url())
14379            .timeout(Duration::from_secs(2))
14380            .build()
14381            .expect("client");
14382        let mut worker = Worker::new(client, "rust-workers")
14383            .worker_id("bounded-retry-worker")
14384            .poll_timeout(Duration::from_millis(10))
14385            .retry_policy(WorkerRetryPolicy {
14386                max_retries: 2,
14387                initial_backoff: Duration::from_millis(1),
14388                max_backoff: Duration::from_millis(1),
14389            });
14390        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14391
14392        let error = worker.run().await.expect_err("retry bound must terminate");
14393        assert!(matches!(error, Error::Transport(_)));
14394        assert_eq!(
14395            server.request_count("/api/worker/workflow-tasks/poll"),
14396            3,
14397            "one initial request plus exactly two retries"
14398        );
14399    }
14400
14401    #[tokio::test]
14402    async fn worker_retry_policy_can_disable_poll_retries() {
14403        let server = MockWorkerServer::unavailable_polls();
14404        let client = Client::builder(server.base_url())
14405            .timeout(Duration::from_secs(2))
14406            .build()
14407            .expect("client");
14408        let mut worker = Worker::new(client, "rust-workers")
14409            .worker_id("no-retry-worker")
14410            .poll_timeout(Duration::from_millis(10))
14411            .retry_policy(WorkerRetryPolicy {
14412                max_retries: 0,
14413                initial_backoff: Duration::from_millis(1),
14414                max_backoff: Duration::from_millis(1),
14415            });
14416        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14417
14418        let error = worker
14419            .run_once()
14420            .await
14421            .expect_err("disabled retries must return the first transport failure");
14422        assert!(matches!(error, Error::Transport(_)));
14423        assert_eq!(
14424            server.request_count("/api/worker/workflow-tasks/poll"),
14425            1,
14426            "max_retries=0 must send only the initial request"
14427        );
14428    }
14429
14430    #[tokio::test]
14431    async fn worker_does_not_retry_authentication_failures() {
14432        let server = MockWorkerServer::unauthorized_polls();
14433        let client = Client::builder(server.base_url())
14434            .timeout(Duration::from_secs(2))
14435            .build()
14436            .expect("client");
14437        let mut worker = Worker::new(client, "rust-workers")
14438            .worker_id("unauthorized-worker")
14439            .poll_timeout(Duration::from_millis(10));
14440        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
14441
14442        let error = worker
14443            .run()
14444            .await
14445            .expect_err("authentication must terminate");
14446        let Error::Http { status, body } = error else {
14447            panic!("expected stable HTTP authentication error");
14448        };
14449        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
14450        assert!(body.contains("authentication_failed"));
14451        assert_eq!(
14452            server.request_count("/api/worker/workflow-tasks/poll"),
14453            1,
14454            "authentication failures must not be retried"
14455        );
14456    }
14457
14458    #[derive(Clone, Debug)]
14459    struct CapturedRequest {
14460        method: String,
14461        path: String,
14462        authorization: Option<String>,
14463        namespace: Option<String>,
14464        worker_protocol: Option<String>,
14465        control_protocol: Option<String>,
14466        body: String,
14467        received_at: Instant,
14468    }
14469
14470    struct MockWorkerServer {
14471        addr: SocketAddr,
14472        stop: Arc<AtomicBool>,
14473        requests: Arc<Mutex<Vec<CapturedRequest>>>,
14474        thread: Option<thread::JoinHandle<()>>,
14475    }
14476
14477    #[derive(Clone, Copy, Default)]
14478    struct MockWorkerBehavior {
14479        reject_query_protocol: bool,
14480        reject_query_completion: bool,
14481        waiting_query_worker: bool,
14482        decline_registration: bool,
14483        complete_named_signal: bool,
14484        poll_failures_per_path: usize,
14485        heartbeat_failures: usize,
14486        heartbeat_failure_request: Option<usize>,
14487        delayed_heartbeat_request: Option<usize>,
14488        heartbeat_response_delay: Duration,
14489        concurrent_requests: bool,
14490        unauthorized_polls: bool,
14491        reject_registration: bool,
14492        reject_deregistration: bool,
14493        reject_deregistration_protocol: bool,
14494        cancelled_activity: bool,
14495        draining_polls: bool,
14496        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
14497        workflow_completion_status: Option<&'static str>,
14498        workflow_completion_body: Option<&'static str>,
14499    }
14500
14501    impl MockWorkerServer {
14502        fn start() -> Self {
14503            Self::start_with_behavior(MockWorkerBehavior::default())
14504        }
14505
14506        fn reject_query_protocol() -> Self {
14507            Self::start_with_behavior(MockWorkerBehavior {
14508                reject_query_protocol: true,
14509                ..MockWorkerBehavior::default()
14510            })
14511        }
14512
14513        fn reject_query_completion() -> Self {
14514            Self::start_with_behavior(MockWorkerBehavior {
14515                reject_query_completion: true,
14516                ..MockWorkerBehavior::default()
14517            })
14518        }
14519
14520        fn waiting_query_worker() -> Self {
14521            Self::start_with_behavior(MockWorkerBehavior {
14522                waiting_query_worker: true,
14523                complete_named_signal: true,
14524                ..MockWorkerBehavior::default()
14525            })
14526        }
14527
14528        fn transient_worker_failures() -> Self {
14529            Self::start_with_behavior(MockWorkerBehavior {
14530                poll_failures_per_path: 1,
14531                heartbeat_failures: 1,
14532                ..MockWorkerBehavior::default()
14533            })
14534        }
14535
14536        fn consecutive_poll_failures(count: usize) -> Self {
14537            Self::start_with_behavior(MockWorkerBehavior {
14538                poll_failures_per_path: count,
14539                ..MockWorkerBehavior::default()
14540            })
14541        }
14542
14543        fn delayed_heartbeat_worker() -> Self {
14544            Self::start_with_behavior(MockWorkerBehavior {
14545                waiting_query_worker: true,
14546                delayed_heartbeat_request: Some(2),
14547                heartbeat_response_delay: Duration::from_millis(1_500),
14548                concurrent_requests: true,
14549                cancelled_activity: true,
14550                ..MockWorkerBehavior::default()
14551            })
14552        }
14553
14554        fn heartbeat_retry_worker() -> Self {
14555            Self::start_with_behavior(MockWorkerBehavior {
14556                waiting_query_worker: true,
14557                heartbeat_failure_request: Some(2),
14558                concurrent_requests: true,
14559                ..MockWorkerBehavior::default()
14560            })
14561        }
14562
14563        fn unavailable_polls() -> Self {
14564            Self::start_with_behavior(MockWorkerBehavior {
14565                poll_failures_per_path: usize::MAX,
14566                ..MockWorkerBehavior::default()
14567            })
14568        }
14569
14570        fn unauthorized_polls() -> Self {
14571            Self::start_with_behavior(MockWorkerBehavior {
14572                unauthorized_polls: true,
14573                ..MockWorkerBehavior::default()
14574            })
14575        }
14576
14577        fn rejected_registration() -> Self {
14578            Self::start_with_behavior(MockWorkerBehavior {
14579                reject_registration: true,
14580                ..MockWorkerBehavior::default()
14581            })
14582        }
14583
14584        fn declined_registration() -> Self {
14585            Self::start_with_behavior(MockWorkerBehavior {
14586                decline_registration: true,
14587                ..MockWorkerBehavior::default()
14588            })
14589        }
14590
14591        fn rejected_deregistration() -> Self {
14592            Self::start_with_behavior(MockWorkerBehavior {
14593                reject_deregistration: true,
14594                ..MockWorkerBehavior::default()
14595            })
14596        }
14597
14598        fn rejected_deregistration_protocol() -> Self {
14599            Self::start_with_behavior(MockWorkerBehavior {
14600                reject_deregistration_protocol: true,
14601                ..MockWorkerBehavior::default()
14602            })
14603        }
14604
14605        fn unauthorized_polls_and_rejected_deregistration() -> Self {
14606            Self::start_with_behavior(MockWorkerBehavior {
14607                unauthorized_polls: true,
14608                reject_deregistration: true,
14609                ..MockWorkerBehavior::default()
14610            })
14611        }
14612
14613        fn cancelled_activity() -> Self {
14614            Self::start_with_behavior(MockWorkerBehavior {
14615                cancelled_activity: true,
14616                ..MockWorkerBehavior::default()
14617            })
14618        }
14619
14620        fn draining_polls() -> Self {
14621            Self::start_with_behavior(MockWorkerBehavior {
14622                draining_polls: true,
14623                ..MockWorkerBehavior::default()
14624            })
14625        }
14626
14627        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
14628            Self::start_with_behavior(MockWorkerBehavior {
14629                invalid_task_payload_codec: Some(codec),
14630                ..MockWorkerBehavior::default()
14631            })
14632        }
14633
14634        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
14635            Self::start_with_behavior(MockWorkerBehavior {
14636                workflow_completion_status: Some(status),
14637                workflow_completion_body: Some(body),
14638                ..MockWorkerBehavior::default()
14639            })
14640        }
14641
14642        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
14643            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
14644            listener
14645                .set_nonblocking(true)
14646                .expect("configure mock listener");
14647            let addr = listener.local_addr().expect("mock server address");
14648            let stop = Arc::new(AtomicBool::new(false));
14649            let server_stop = Arc::clone(&stop);
14650            let requests = Arc::new(Mutex::new(Vec::new()));
14651            let server_requests = Arc::clone(&requests);
14652            let thread = thread::spawn(move || {
14653                let mut request_threads = Vec::new();
14654                while !server_stop.load(Ordering::SeqCst) {
14655                    match listener.accept() {
14656                        Ok((mut stream, _)) => {
14657                            if behavior.concurrent_requests {
14658                                let requests = Arc::clone(&server_requests);
14659                                request_threads.push(thread::spawn(move || {
14660                                    handle_mock_worker_request(&mut stream, &requests, behavior)
14661                                }));
14662                            } else {
14663                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
14664                            }
14665                        }
14666                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
14667                            let mut index = 0;
14668                            while index < request_threads.len() {
14669                                if request_threads[index].is_finished() {
14670                                    request_threads
14671                                        .swap_remove(index)
14672                                        .join()
14673                                        .expect("join mock request");
14674                                } else {
14675                                    index += 1;
14676                                }
14677                            }
14678                            thread::sleep(Duration::from_millis(5));
14679                        }
14680                        Err(_) => break,
14681                    }
14682                }
14683                for request_thread in request_threads {
14684                    request_thread.join().expect("join mock request");
14685                }
14686            });
14687
14688            Self {
14689                addr,
14690                stop,
14691                requests,
14692                thread: Some(thread),
14693            }
14694        }
14695
14696        fn base_url(&self) -> String {
14697            format!("http://{}", self.addr)
14698        }
14699
14700        fn worker_protocol_for(&self, path: &str) -> Option<String> {
14701            self.requests
14702                .lock()
14703                .expect("captured requests")
14704                .iter()
14705                .find(|request| request.path == path)
14706                .and_then(|request| request.worker_protocol.clone())
14707        }
14708
14709        fn control_protocol_for(&self, path: &str) -> Option<String> {
14710            self.requests
14711                .lock()
14712                .expect("captured requests")
14713                .iter()
14714                .find(|request| request.path == path)
14715                .and_then(|request| request.control_protocol.clone())
14716        }
14717
14718        fn method_for(&self, path: &str) -> Option<String> {
14719            self.requests
14720                .lock()
14721                .expect("captured requests")
14722                .iter()
14723                .find(|request| request.path == path)
14724                .map(|request| request.method.clone())
14725        }
14726
14727        fn authorization_for(&self, path: &str) -> Option<String> {
14728            self.requests
14729                .lock()
14730                .expect("captured requests")
14731                .iter()
14732                .find(|request| request.path == path)
14733                .and_then(|request| request.authorization.clone())
14734        }
14735
14736        fn namespace_for(&self, path: &str) -> Option<String> {
14737            self.requests
14738                .lock()
14739                .expect("captured requests")
14740                .iter()
14741                .find(|request| request.path == path)
14742                .and_then(|request| request.namespace.clone())
14743        }
14744
14745        fn request_count(&self, path: &str) -> usize {
14746            self.requests
14747                .lock()
14748                .expect("captured requests")
14749                .iter()
14750                .filter(|request| request.path == path)
14751                .count()
14752        }
14753
14754        fn captured_paths(&self) -> Vec<String> {
14755            self.requests
14756                .lock()
14757                .expect("captured requests")
14758                .iter()
14759                .map(|request| request.path.clone())
14760                .collect()
14761        }
14762
14763        fn request_times(&self, path: &str) -> Vec<Instant> {
14764            self.requests
14765                .lock()
14766                .expect("captured requests")
14767                .iter()
14768                .filter(|request| request.path == path)
14769                .map(|request| request.received_at)
14770                .collect()
14771        }
14772
14773        fn request_body(&self, path: &str) -> Value {
14774            let requests = self.requests.lock().expect("captured requests");
14775            let body = &requests
14776                .iter()
14777                .find(|request| request.path == path)
14778                .unwrap_or_else(|| panic!("missing request for {path}"))
14779                .body;
14780            serde_json::from_str(body).unwrap_or_else(|error| {
14781                panic!("invalid JSON request body for {path}: {error}: {body:?}")
14782            })
14783        }
14784
14785        fn request_bodies(&self, path: &str) -> Vec<Value> {
14786            self.requests
14787                .lock()
14788                .expect("captured requests")
14789                .iter()
14790                .filter(|request| request.path == path)
14791                .map(|request| {
14792                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
14793                        panic!(
14794                            "invalid JSON request body for {path}: {error}: {:?}",
14795                            request.body
14796                        )
14797                    })
14798                })
14799                .collect()
14800        }
14801    }
14802
14803    impl Drop for MockWorkerServer {
14804        fn drop(&mut self) {
14805            self.stop.store(true, Ordering::SeqCst);
14806            let _ = TcpStream::connect(self.addr);
14807
14808            if let Some(thread) = self.thread.take() {
14809                thread.join().expect("join mock server");
14810            }
14811        }
14812    }
14813
14814    fn handle_mock_worker_request(
14815        stream: &mut TcpStream,
14816        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
14817        behavior: MockWorkerBehavior,
14818    ) {
14819        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
14820        let mut buffer = [0_u8; 8192];
14821        let mut request = Vec::new();
14822
14823        loop {
14824            match stream.read(&mut buffer) {
14825                Ok(0) => break,
14826                Ok(read) => {
14827                    request.extend_from_slice(&buffer[..read]);
14828                    if mock_request_is_complete(&request) {
14829                        break;
14830                    }
14831                }
14832                Err(error)
14833                    if matches!(
14834                        error.kind(),
14835                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
14836                    ) =>
14837                {
14838                    break;
14839                }
14840                Err(_) => return,
14841            }
14842        }
14843
14844        let request = String::from_utf8_lossy(&request);
14845        let body = request
14846            .split_once("\r\n\r\n")
14847            .map(|(_, body)| body)
14848            .unwrap_or_default();
14849        let path = request
14850            .lines()
14851            .next()
14852            .and_then(|line| line.split_whitespace().nth(1))
14853            .unwrap_or_default();
14854        let method = request
14855            .lines()
14856            .next()
14857            .and_then(|line| line.split_whitespace().next())
14858            .unwrap_or_default();
14859        let authorization = request.lines().find_map(|line| {
14860            let (name, value) = line.split_once(':')?;
14861            name.eq_ignore_ascii_case("Authorization")
14862                .then(|| value.trim().to_string())
14863        });
14864        let namespace = request.lines().find_map(|line| {
14865            let (name, value) = line.split_once(':')?;
14866            name.eq_ignore_ascii_case("X-Namespace")
14867                .then(|| value.trim().to_string())
14868        });
14869        let worker_protocol = request.lines().find_map(|line| {
14870            let (name, value) = line.split_once(':')?;
14871            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
14872                .then(|| value.trim().to_string())
14873        });
14874        let control_protocol = request.lines().find_map(|line| {
14875            let (name, value) = line.split_once(':')?;
14876            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
14877                .then(|| value.trim().to_string())
14878        });
14879        let request_number = {
14880            let mut requests = requests.lock().expect("captured requests");
14881            requests.push(CapturedRequest {
14882                method: method.to_string(),
14883                path: path.to_string(),
14884                authorization,
14885                namespace,
14886                worker_protocol: worker_protocol.clone(),
14887                control_protocol,
14888                body: body.to_string(),
14889                received_at: Instant::now(),
14890            });
14891            requests
14892                .iter()
14893                .filter(|request| request.path == path)
14894                .count()
14895        };
14896
14897        if behavior.reject_registration && path == "/api/worker/register" {
14898            write_mock_response(
14899                stream,
14900                "503 Service Unavailable",
14901                r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
14902            );
14903            return;
14904        }
14905
14906        if path.starts_with("/api/worker/registrations/") {
14907            if behavior.reject_deregistration_protocol {
14908                write_mock_response(
14909                    stream,
14910                    "400 Bad Request",
14911                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
14912                );
14913            } else if behavior.reject_deregistration {
14914                write_mock_response(
14915                    stream,
14916                    "403 Forbidden",
14917                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
14918                );
14919            } else {
14920                write_mock_response(
14921                    stream,
14922                    "200 OK",
14923                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
14924                );
14925            }
14926            return;
14927        }
14928
14929        let is_poll = matches!(
14930            path,
14931            "/api/worker/workflow-tasks/poll"
14932                | "/api/worker/activity-tasks/poll"
14933                | "/api/worker/query-tasks/poll"
14934        );
14935        if is_poll && request_number <= behavior.poll_failures_per_path {
14936            return;
14937        }
14938        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14939            return;
14940        }
14941        if path == "/api/worker/heartbeat"
14942            && behavior.heartbeat_failure_request == Some(request_number)
14943        {
14944            return;
14945        }
14946        if path == "/api/worker/heartbeat"
14947            && behavior.delayed_heartbeat_request == Some(request_number)
14948        {
14949            thread::sleep(behavior.heartbeat_response_delay);
14950        }
14951        if behavior.unauthorized_polls && is_poll {
14952            write_mock_response(
14953                stream,
14954                "401 Unauthorized",
14955                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14956            );
14957            return;
14958        }
14959        if behavior.draining_polls && is_poll {
14960            write_mock_response(
14961                stream,
14962                "409 Conflict",
14963                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14964            );
14965            return;
14966        }
14967
14968        if let Some(codec_case) = behavior.invalid_task_payload_codec {
14969            if is_poll && request_number == 1 {
14970                let mut task = match path {
14971                    "/api/worker/workflow-tasks/poll" => json!({
14972                        "task_id": "codec-workflow",
14973                        "workflow_type": "codec.workflow",
14974                        "payload_codec": DEFAULT_CODEC,
14975                        "workflow_task_attempt": 1,
14976                        "lease_owner": "codec-worker"
14977                    }),
14978                    "/api/worker/activity-tasks/poll" => json!({
14979                        "task_id": "codec-activity",
14980                        "activity_attempt_id": "codec-activity-attempt",
14981                        "activity_type": "codec.activity",
14982                        "payload_codec": DEFAULT_CODEC,
14983                        "attempt_number": 1,
14984                        "lease_owner": "codec-worker"
14985                    }),
14986                    "/api/worker/query-tasks/poll" => json!({
14987                        "query_task_id": "codec-query",
14988                        "query_task_attempt": 1,
14989                        "workflow_type": "codec.workflow",
14990                        "query_name": "known",
14991                        "payload_codec": DEFAULT_CODEC,
14992                        "lease_owner": "codec-worker"
14993                    }),
14994                    _ => unreachable!("is_poll limits task codec probe paths"),
14995                };
14996                codec_case.apply(&mut task);
14997                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
14998                return;
14999            }
15000
15001            if matches!(
15002                path,
15003                "/api/worker/workflow-tasks/codec-workflow/fail"
15004                    | "/api/worker/activity-tasks/codec-activity/fail"
15005                    | "/api/worker/query-tasks/codec-query/fail"
15006            ) {
15007                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
15008                return;
15009            }
15010        }
15011
15012        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
15013            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
15014            let body = format!(
15015                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
15016            );
15017            write_mock_response(stream, "400 Bad Request", &body);
15018            return;
15019        }
15020
15021        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
15022        {
15023            write_mock_response(
15024                stream,
15025                "409 Conflict",
15026                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
15027            );
15028            return;
15029        }
15030
15031        if behavior.workflow_completion_status.is_some()
15032            && path == "/api/worker/workflow-tasks/poll"
15033            && request_number == 1
15034        {
15035            write_mock_response(
15036                stream,
15037                "200 OK",
15038                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
15039            );
15040            return;
15041        }
15042
15043        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
15044            if let (Some(status), Some(body)) = (
15045                behavior.workflow_completion_status,
15046                behavior.workflow_completion_body,
15047            ) {
15048                write_mock_response(stream, status, body);
15049                return;
15050            }
15051        }
15052
15053        if behavior.waiting_query_worker {
15054            if behavior.complete_named_signal
15055                && path == "/api/worker/workflow-tasks/poll"
15056                && request_number == 1
15057            {
15058                let body = json!({
15059                    "task": {
15060                        "task_id": "snapshot-open",
15061                        "workflow_id": "snapshot-1",
15062                        "run_id": "snapshot-run-1",
15063                        "workflow_type": "snapshot",
15064                        "payload_codec": DEFAULT_CODEC,
15065                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15066                            .expect("Avro workflow arguments"),
15067                        "history_events": [],
15068                        "workflow_task_attempt": 1,
15069                        "lease_owner": "rust-snapshot-worker"
15070                    }
15071                })
15072                .to_string();
15073                write_mock_response(stream, "200 OK", &body);
15074                return;
15075            }
15076
15077            let signal_request = request_number - usize::from(behavior.complete_named_signal);
15078            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
15079            if path == "/api/worker/workflow-tasks/poll"
15080                && signal_request >= 1
15081                && signal_request <= signal_request_limit
15082            {
15083                let finish = behavior.complete_named_signal && signal_request == 3;
15084                let amounts = if signal_request == 1 {
15085                    vec![3]
15086                } else {
15087                    vec![3, 5]
15088                };
15089                let task_id = if signal_request == 1 {
15090                    "snapshot-wait-3"
15091                } else if finish {
15092                    "snapshot-finish"
15093                } else {
15094                    "snapshot-wait-5"
15095                };
15096                let mut history_events = std::iter::once(json!({
15097                    "event_type": "SignalWaitOpened",
15098                    "payload": {"sequence": 1, "signal_name": "finish"}
15099                }))
15100                .chain(amounts.iter().enumerate().map(|(index, amount)| {
15101                    json!({
15102                        "event_type": "SignalReceived",
15103                        "payload": {
15104                            "signal_id": format!("increment-{amount}"),
15105                            "signal_name": "increment",
15106                            "workflow_sequence": index + 2,
15107                            "payload_codec": DEFAULT_CODEC,
15108                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
15109                                .expect("Avro signal envelope")
15110                        }
15111                    })
15112                }))
15113                .collect::<Vec<_>>();
15114                let (resume_id, resume_name, resume_arguments) = if finish {
15115                    history_events.push(json!({
15116                        "event_type": "SignalReceived",
15117                        "payload": {
15118                            "signal_id": "finish",
15119                            "signal_name": "finish",
15120                            "workflow_sequence": 4,
15121                            "payload_codec": DEFAULT_CODEC,
15122                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15123                                .expect("Avro finish signal envelope")
15124                        }
15125                    }));
15126                    (
15127                        "finish".to_string(),
15128                        "finish".to_string(),
15129                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
15130                            .expect("Avro finish resume signal"),
15131                    )
15132                } else {
15133                    let amount = amounts.last().expect("amount");
15134                    (
15135                        format!("increment-{amount}"),
15136                        "increment".to_string(),
15137                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
15138                            .expect("Avro increment resume signal"),
15139                    )
15140                };
15141                let body = json!({
15142                    "task": {
15143                        "task_id": task_id,
15144                        "workflow_id": "snapshot-1",
15145                        "run_id": "snapshot-run-1",
15146                        "workflow_type": "snapshot",
15147                        "payload_codec": DEFAULT_CODEC,
15148                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15149                            .expect("Avro workflow arguments"),
15150                        "history_events": history_events,
15151                        "workflow_task_attempt": 1,
15152                        "workflow_signal_id": resume_id,
15153                        "signal_name": resume_name,
15154                        "signal_arguments": resume_arguments,
15155                        "lease_owner": "rust-snapshot-worker"
15156                    }
15157                })
15158                .to_string();
15159                write_mock_response(stream, "200 OK", &body);
15160                return;
15161            }
15162
15163            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
15164                let history_events = [3, 5]
15165                    .into_iter()
15166                    .enumerate()
15167                    .map(|(index, amount)| {
15168                        json!({
15169                            "event_type": "SignalReceived",
15170                            "payload": {
15171                                "signal_id": format!("increment-{amount}"),
15172                                "signal_name": "increment",
15173                                "workflow_sequence": index + 2,
15174                                "payload_codec": DEFAULT_CODEC,
15175                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
15176                                    .expect("Avro query signal envelope")
15177                            }
15178                        })
15179                    })
15180                    .collect::<Vec<_>>();
15181                let body = json!({
15182                    "task": {
15183                        "query_task_id": "snapshot-current",
15184                        "query_task_attempt": 1,
15185                        "lease_owner": "rust-snapshot-worker",
15186                        "workflow_id": "snapshot-1",
15187                        "run_id": "snapshot-run-1",
15188                        "workflow_type": "snapshot",
15189                        "query_name": "current",
15190                        "payload_codec": DEFAULT_CODEC,
15191                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15192                            .expect("Avro workflow arguments"),
15193                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
15194                            .expect("Avro query arguments"),
15195                        "history_events": history_events,
15196                        "run_status": "waiting"
15197                    }
15198                })
15199                .to_string();
15200                write_mock_response(stream, "200 OK", &body);
15201                return;
15202            }
15203
15204            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
15205                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
15206            {
15207                write_mock_response(
15208                    stream,
15209                    "200 OK",
15210                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
15211                );
15212                return;
15213            }
15214
15215            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
15216                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
15217                return;
15218            }
15219
15220            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
15221                write_mock_response(
15222                    stream,
15223                    "200 OK",
15224                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
15225                );
15226                return;
15227            }
15228
15229            if path == "/api/worker/query-tasks/snapshot-current/complete" {
15230                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
15231                return;
15232            }
15233        }
15234
15235        if matches!(
15236            path,
15237            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
15238        ) {
15239            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
15240                .expect("typed mock result");
15241            let body = json!({
15242                "result": typed_fidelity_probe().into_json().expect("result projection"),
15243                "result_envelope": result,
15244            })
15245            .to_string();
15246            write_mock_response(stream, "200 OK", &body);
15247            return;
15248        }
15249
15250        if path == "/api/workflows/typed-1" {
15251            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
15252                .expect("typed mock result");
15253            let body = json!({
15254                "workflow_id": "typed-1",
15255                "run_id": "run-typed-1",
15256                "workflow_type": "typed.echo",
15257                "status": "completed",
15258                "output": typed_fidelity_probe().into_json().expect("output projection"),
15259                "output_envelope": result,
15260            })
15261            .to_string();
15262            write_mock_response(stream, "200 OK", &body);
15263            return;
15264        }
15265
15266        let (status, body) = match path {
15267            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
15268            "/api/workflows" => (
15269                "201 Created",
15270                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
15271            ),
15272            "/api/worker/register" if behavior.decline_registration => (
15273                "200 OK",
15274                r#"{"worker_id":"declined-worker","registered":false}"#,
15275            ),
15276            "/api/worker/register" if behavior.waiting_query_worker => (
15277                "200 OK",
15278                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
15279            ),
15280            "/api/worker/register" => (
15281                "200 OK",
15282                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
15283            ),
15284            "/api/worker/heartbeat" => ("200 OK", "{}"),
15285            "/api/worker/activity-tasks/poll"
15286                if behavior.cancelled_activity && request_number == 1 =>
15287            {
15288                (
15289                    "200 OK",
15290                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
15291                )
15292            }
15293            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
15294                ("200 OK", r#"{"task":null}"#)
15295            }
15296            "/api/worker/query-tasks/poll"
15297                if behavior.reject_query_completion && request_number == 1 =>
15298            {
15299                (
15300                    "200 OK",
15301                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
15302                )
15303            }
15304            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
15305            "/api/worker/query-tasks/query-capture/complete"
15306            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
15307            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
15308                "200 OK",
15309                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
15310            ),
15311            "/api/worker/activity-tasks/activity-cancel/complete" => (
15312                "409 Conflict",
15313                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
15314            ),
15315            "/api/worker/activity-tasks/activity-typed/complete"
15316            | "/api/worker/activity-tasks/activity-typed/fail"
15317            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
15318            "/api/workflows/counter-1/query/current" => (
15319                "200 OK",
15320                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
15321            ),
15322            "/api/workflows/counter-1/query/missing" => (
15323                "404 Not Found",
15324                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
15325            ),
15326            "/api/workflows/wf-lifecycle/cancel" => (
15327                "200 OK",
15328                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
15329            ),
15330            "/api/workflows/wf-lifecycle/terminate" => (
15331                "200 OK",
15332                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
15333            ),
15334            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
15335                "200 OK",
15336                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
15337            ),
15338            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
15339                "200 OK",
15340                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
15341            ),
15342            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
15343            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
15344                "409 Conflict",
15345                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
15346            ),
15347            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
15348                "200 OK",
15349                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
15350            ),
15351            "/api/workflows/wf-cancelled" => (
15352                "200 OK",
15353                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
15354            ),
15355            "/api/workflows/wf-terminated" => (
15356                "200 OK",
15357                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
15358            ),
15359            "/api/workflows/wf-timed-out" => (
15360                "200 OK",
15361                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
15362            ),
15363            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
15364                "200 OK",
15365                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
15366            ),
15367            "/api/workflows/wf-selected" => (
15368                "200 OK",
15369                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
15370            ),
15371            "/api/workflows/wf-selected/runs/run-selected" => (
15372                "200 OK",
15373                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
15374            ),
15375            _ => ("404 Not Found", r#"{"message":"not found"}"#),
15376        };
15377        write_mock_response(stream, status, body);
15378    }
15379
15380    fn mock_request_is_complete(request: &[u8]) -> bool {
15381        let Some(header_end) = request
15382            .windows(4)
15383            .position(|window| window == b"\r\n\r\n")
15384            .map(|position| position + 4)
15385        else {
15386            return false;
15387        };
15388        let headers = String::from_utf8_lossy(&request[..header_end]);
15389        let content_length = headers.lines().find_map(|line| {
15390            let (name, value) = line.split_once(':')?;
15391            name.eq_ignore_ascii_case("content-length")
15392                .then(|| value.trim().parse::<usize>().ok())
15393                .flatten()
15394        });
15395
15396        request.len() >= header_end + content_length.unwrap_or(0)
15397    }
15398
15399    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
15400        let response = format!(
15401            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
15402            body.len()
15403        );
15404
15405        let _ = stream.write_all(response.as_bytes());
15406        let _ = stream.flush();
15407    }
15408}