Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21    de::DeserializeOwned,
22    ser::{SerializeMap, SerializeSeq},
23    Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34/// Worker-registration capability for server-routed read-only queries.
35pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36/// Worker-registration capability for synchronous workflow updates.
37pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38/// First additive worker protocol that defines query-task transport.
39pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43    "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47    "lease_expired",
48    "query_task_not_found",
49    "query_task_not_leased",
50    "query_task_timed_out",
51];
52
53/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
54pub const AVRO_VALUE_SCHEMA_JSON: &str =
55    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64    ControlPlane,
65    Worker(&'static str),
66}
67
68pub type Result<T> = std::result::Result<T, Error>;
69
70#[derive(Debug, Error)]
71pub enum Error {
72    #[error("transport error: {0}")]
73    Transport(#[from] reqwest::Error),
74    #[error(
75        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
76    )]
77    InvalidBaseUrl,
78    #[error("json error: {0}")]
79    Json(#[from] serde_json::Error),
80    #[error("http {status}: {body}")]
81    Http {
82        status: reqwest::StatusCode,
83        body: String,
84    },
85    #[error("codec error: {0}")]
86    Codec(String),
87    #[error(transparent)]
88    QueryFailed(QueryFailure),
89    #[error(transparent)]
90    Protocol(ProtocolFailure),
91    #[error(transparent)]
92    NonDeterministicReplay(ReplayFailure),
93    #[error(transparent)]
94    ChildWorkflowFailed(ChildWorkflowFailure),
95    #[error(transparent)]
96    ActivityFailed(ActivityFailure),
97    #[error(transparent)]
98    WorkflowCommandRejected(WorkflowCommandRejection),
99    #[error(transparent)]
100    WorkflowFailed(WorkflowTerminalOutcome),
101    #[error(transparent)]
102    WorkflowCancelled(WorkflowTerminalOutcome),
103    #[error(transparent)]
104    WorkflowTerminated(WorkflowTerminalOutcome),
105    #[error(transparent)]
106    WorkflowTimedOut(WorkflowTerminalOutcome),
107    #[error(transparent)]
108    ActivityTaskRejected(ActivityTaskRejection),
109    #[error("workflow handler {0:?} is not registered")]
110    WorkflowNotRegistered(String),
111    #[error("activity handler {0:?} is not registered")]
112    ActivityNotRegistered(String),
113    #[error("workflow future yielded without emitting a durable command")]
114    WorkflowYieldedWithoutCommand,
115    #[error("workflow state lock is poisoned")]
116    WorkflowStatePoisoned,
117    #[error("timer duration is too large for the worker protocol")]
118    TimerDurationOverflow,
119    #[error("operation timed out")]
120    Timeout,
121    #[error(
122        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
123    )]
124    MissingRoleCredentials {
125        role: &'static str,
126        opposite_role: &'static str,
127    },
128    #[error("worker loop error: {0}")]
129    WorkerLoop(String),
130    #[error("{primary}; worker deregistration also failed: {deregistration}")]
131    WorkerShutdown {
132        primary: Box<Error>,
133        deregistration: Box<Error>,
134    },
135    #[error("invalid child workflow options: {0}")]
136    InvalidChildWorkflowOptions(String),
137    #[error(transparent)]
138    InvalidActivityOptions(ActivityOptionsError),
139    #[error(transparent)]
140    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
141    #[doc(hidden)]
142    #[error("workflow requested continue as new")]
143    ContinueAsNew(ContinueAsNewRequest),
144}
145
146/// The lifecycle command sent to a workflow execution.
147#[derive(Clone, Copy, Debug, PartialEq, Eq)]
148pub enum WorkflowCommandKind {
149    Cancel,
150    Terminate,
151}
152
153impl WorkflowCommandKind {
154    fn as_str(self) -> &'static str {
155        match self {
156            Self::Cancel => "cancel",
157            Self::Terminate => "terminate",
158        }
159    }
160}
161
162/// Optional structured fields for a cancellation or termination request.
163#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
164pub struct WorkflowCommandOptions {
165    #[serde(skip_serializing_if = "Option::is_none")]
166    pub reason: Option<String>,
167    #[serde(skip_serializing_if = "Option::is_none")]
168    pub request_id: Option<String>,
169}
170
171/// Server-enforced timeout policy for a workflow start.
172///
173/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
174/// only bounds how long the caller waits. A server deadline produces a terminal
175/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
176/// `run_timeout`.
177#[derive(Clone, Debug, PartialEq, Eq)]
178pub struct WorkflowStartOptions {
179    pub execution_timeout_seconds: u64,
180    pub run_timeout_seconds: u64,
181}
182
183impl Default for WorkflowStartOptions {
184    fn default() -> Self {
185        Self {
186            execution_timeout_seconds: 3600,
187            run_timeout_seconds: 600,
188        }
189    }
190}
191
192impl WorkflowStartOptions {
193    pub fn new() -> Self {
194        Self::default()
195    }
196
197    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
198        self.execution_timeout_seconds = seconds;
199        self
200    }
201
202    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
203        self.run_timeout_seconds = seconds;
204        self
205    }
206
207    fn validate(&self) -> Result<()> {
208        if self.execution_timeout_seconds == 0 {
209            return Err(Error::Codec(
210                "execution_timeout_seconds must be at least 1".to_string(),
211            ));
212        }
213        if self.run_timeout_seconds == 0 {
214            return Err(Error::Codec(
215                "run_timeout_seconds must be at least 1".to_string(),
216            ));
217        }
218        if self.run_timeout_seconds > self.execution_timeout_seconds {
219            return Err(Error::Codec(
220                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
221            ));
222        }
223
224        Ok(())
225    }
226}
227
228/// Optional routing overrides for a continue-as-new transition.
229///
230/// Omitted values retain the current workflow type and task queue. Server-owned
231/// instance metadata is not accepted here and is carried by the server.
232#[derive(Clone, Debug, Default, PartialEq, Eq)]
233pub struct ContinueAsNewOptions {
234    pub workflow_type: Option<String>,
235    pub task_queue: Option<String>,
236}
237
238impl ContinueAsNewOptions {
239    pub fn new() -> Self {
240        Self::default()
241    }
242
243    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
244        self.workflow_type = Some(workflow_type.into());
245        self
246    }
247
248    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
249        self.task_queue = Some(task_queue.into());
250        self
251    }
252
253    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
254        for (field, value) in [
255            ("workflow_type", self.workflow_type.as_deref()),
256            ("task_queue", self.task_queue.as_deref()),
257        ] {
258            if value.is_some_and(|value| value.trim().is_empty()) {
259                return Err(ContinueAsNewOptionsError {
260                    field,
261                    message: format!("{field} must not be empty"),
262                });
263            }
264        }
265        Ok(())
266    }
267}
268
269/// A stable validation error raised before a continue-as-new command is emitted.
270#[derive(Clone, Debug, Error, PartialEq, Eq)]
271#[error("invalid continue-as-new option {field}: {message}")]
272pub struct ContinueAsNewOptionsError {
273    pub field: &'static str,
274    pub message: String,
275}
276
277/// Public history-budget information attached to the current workflow task.
278#[derive(Clone, Debug, Default, PartialEq, Eq)]
279pub struct WorkflowHistoryBudget {
280    pub event_count: u64,
281    pub size_bytes: Option<u64>,
282    pub continue_as_new_recommended: bool,
283    pub pressure: Option<String>,
284}
285
286#[doc(hidden)]
287#[derive(Clone, Debug)]
288pub struct ContinueAsNewRequest {
289    arguments: AvroValue,
290    options: ContinueAsNewOptions,
291}
292
293impl WorkflowCommandOptions {
294    pub fn new() -> Self {
295        Self::default()
296    }
297
298    pub fn reason(mut self, reason: impl Into<String>) -> Self {
299        self.reason = Some(reason.into());
300        self
301    }
302
303    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
304        self.request_id = Some(request_id.into());
305        self
306    }
307}
308
309/// The accepted, machine-readable result of a lifecycle command.
310#[derive(Clone, Debug, PartialEq)]
311pub struct WorkflowCommandResult {
312    pub command: WorkflowCommandKind,
313    pub workflow_id: String,
314    pub run_id: Option<String>,
315    pub outcome: Option<String>,
316    pub reason: Option<String>,
317    pub command_status: Option<String>,
318    pub raw: Value,
319}
320
321/// A stable rejection returned by instance- or selected-run lifecycle commands.
322#[derive(Clone, Debug, Error)]
323#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
324pub struct WorkflowCommandRejection {
325    pub command: WorkflowCommandKind,
326    pub status: u16,
327    pub reason: String,
328    pub message: String,
329    pub workflow_id: String,
330    pub run_id: Option<String>,
331    pub target_scope: Option<String>,
332    pub body: Value,
333}
334
335/// Stable terminal categories returned by [`WorkflowHandle::result`].
336#[derive(Clone, Copy, Debug, PartialEq, Eq)]
337pub enum WorkflowTerminalKind {
338    Failed,
339    Cancelled,
340    Terminated,
341    TimedOut,
342}
343
344/// A typed terminal workflow outcome with durable identity and failure metadata.
345///
346/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
347/// of parsing its display representation. Fields remain `None` when an older
348/// server did not publish that metadata.
349#[derive(Clone, Debug, Error)]
350#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
351pub struct WorkflowTerminalOutcome {
352    pub kind: WorkflowTerminalKind,
353    pub workflow_id: String,
354    pub run_id: Option<String>,
355    pub reason: String,
356    pub failure_category: Option<String>,
357    pub failure_id: Option<String>,
358    pub exception_type: Option<String>,
359    pub exception_class: Option<String>,
360    pub non_retryable: Option<bool>,
361    pub message: Option<String>,
362    pub exception: Option<Value>,
363    pub raw: Value,
364}
365
366/// A worker-side activity settlement or heartbeat rejected by durable state.
367#[derive(Clone, Debug, Error)]
368#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
369pub struct ActivityTaskRejection {
370    pub operation: String,
371    pub status: u16,
372    pub reason: String,
373    pub task_id: String,
374    pub activity_attempt_id: String,
375    pub cancel_requested: bool,
376    pub can_continue: Option<bool>,
377    pub run_closed_reason: Option<String>,
378    pub body: Value,
379}
380
381/// Stable validation categories for [`ActivityOptions`].
382#[derive(Clone, Copy, Debug, PartialEq, Eq)]
383pub enum ActivityOptionsErrorKind {
384    EmptyTaskQueue,
385    EmptyRetryPolicy,
386    InvalidMaxAttempts,
387    BackoffWithoutRetryBudget,
388    TooManyBackoffIntervals,
389    InvalidBackoffCoefficient,
390    BackoffGenerationTooLarge,
391    BackoffOverflow,
392    EmptyNonRetryableErrorType,
393    TimeoutNotPositive,
394    TimeoutOverflow,
395    TimeoutOrder,
396}
397
398/// A machine-readable activity-options validation failure.
399#[derive(Clone, Debug, Error, PartialEq, Eq)]
400#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
401pub struct ActivityOptionsError {
402    pub kind: ActivityOptionsErrorKind,
403    pub field: Option<&'static str>,
404    pub message: String,
405}
406
407impl ActivityOptionsError {
408    fn new(
409        kind: ActivityOptionsErrorKind,
410        field: Option<&'static str>,
411        message: impl Into<String>,
412    ) -> Self {
413        Self {
414            kind,
415            field,
416            message: message.into(),
417        }
418    }
419}
420
421/// Stable terminal categories returned when an awaited activity does not succeed.
422#[derive(Clone, Copy, Debug, PartialEq, Eq)]
423pub enum ActivityFailureKind {
424    Failed,
425    Cancelled,
426    TimedOut,
427}
428
429/// A stable, machine-readable terminal activity failure.
430///
431/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
432/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
433#[derive(Clone, Debug, Error)]
434#[error("activity failed ({reason}): {message}")]
435pub struct ActivityFailure {
436    pub kind: ActivityFailureKind,
437    pub reason: String,
438    pub message: String,
439    pub activity_execution_id: Option<String>,
440    pub activity_attempt_id: Option<String>,
441    pub activity_type: Option<String>,
442    pub activity_class: Option<String>,
443    pub attempt_number: Option<u64>,
444    pub failure_id: Option<String>,
445    pub failure_category: Option<String>,
446    pub timeout_kind: Option<String>,
447    pub non_retryable: bool,
448    pub exception_type: Option<String>,
449    pub exception_class: Option<String>,
450    pub code: Option<Value>,
451    pub exception: Option<Value>,
452}
453
454/// Stable terminal categories returned when an awaited child does not succeed.
455#[derive(Clone, Copy, Debug, PartialEq, Eq)]
456pub enum ChildWorkflowFailureKind {
457    Failed,
458    Cancelled,
459    Terminated,
460}
461
462/// A stable, machine-readable child workflow failure delivered to its parent.
463///
464/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
465/// of parsing the display message. Child and parent identifiers retain the
466/// relationship recorded in durable history across worker restarts.
467#[derive(Clone, Debug, Error)]
468#[error("child workflow failed ({reason}): {message}")]
469pub struct ChildWorkflowFailure {
470    pub kind: ChildWorkflowFailureKind,
471    pub reason: String,
472    pub message: String,
473    pub parent_workflow_id: Option<String>,
474    pub parent_workflow_run_id: Option<String>,
475    pub child_workflow_id: Option<String>,
476    pub child_workflow_run_id: Option<String>,
477    pub child_workflow_type: Option<String>,
478    pub failure_id: Option<String>,
479    pub failure_category: Option<String>,
480    pub exception_type: Option<String>,
481    pub exception_class: Option<String>,
482    pub non_retryable: bool,
483    pub code: Option<Value>,
484    pub exception: Option<Value>,
485}
486
487/// The identity of one durable workflow execution.
488#[derive(Clone, Debug, PartialEq, Eq)]
489pub struct WorkflowIdentity {
490    pub workflow_id: Option<String>,
491    pub run_id: Option<String>,
492}
493
494/// A successful child result together with its durable parent-child identity.
495#[derive(Clone, Debug, PartialEq)]
496pub struct ChildWorkflowResult {
497    pub parent: WorkflowIdentity,
498    pub child: WorkflowIdentity,
499    pub child_workflow_type: Option<String>,
500    pub result: Value,
501}
502
503/// Lossless successful child result for fixed Avro Value workflows.
504#[derive(Clone, Debug, PartialEq)]
505pub struct ChildWorkflowAvroResult {
506    pub parent: WorkflowIdentity,
507    pub child: WorkflowIdentity,
508    pub child_workflow_type: Option<String>,
509    pub result: AvroValue,
510}
511
512/// Server behavior when a parent closes while its child is still open.
513#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
514pub enum ParentClosePolicy {
515    #[default]
516    Abandon,
517    RequestCancel,
518    Terminate,
519}
520
521impl ParentClosePolicy {
522    fn as_str(self) -> &'static str {
523        match self {
524            Self::Abandon => "abandon",
525            Self::RequestCancel => "request_cancel",
526            Self::Terminate => "terminate",
527        }
528    }
529}
530
531/// Durable retry policy for one child workflow invocation.
532#[derive(Clone, Debug, Default, PartialEq, Eq)]
533pub struct ChildWorkflowRetryPolicy {
534    pub max_attempts: Option<u32>,
535    pub backoff_seconds: Vec<u64>,
536    pub non_retryable_error_types: Vec<String>,
537}
538
539/// Options recorded with a child-workflow command.
540///
541/// The task queue is mandatory so routing is explicit and replay-stable.
542#[derive(Clone, Debug, PartialEq, Eq)]
543pub struct ChildWorkflowOptions {
544    pub task_queue: String,
545    pub parent_close_policy: ParentClosePolicy,
546    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
547    pub execution_timeout_seconds: Option<u64>,
548    pub run_timeout_seconds: Option<u64>,
549}
550
551impl ChildWorkflowOptions {
552    pub fn new(task_queue: impl Into<String>) -> Self {
553        Self {
554            task_queue: task_queue.into(),
555            parent_close_policy: ParentClosePolicy::Abandon,
556            retry_policy: None,
557            execution_timeout_seconds: None,
558            run_timeout_seconds: None,
559        }
560    }
561
562    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
563        self.parent_close_policy = policy;
564        self
565    }
566
567    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
568        self.retry_policy = Some(policy);
569        self
570    }
571
572    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
573        self.execution_timeout_seconds = Some(seconds);
574        self
575    }
576
577    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
578        self.run_timeout_seconds = Some(seconds);
579        self
580    }
581}
582
583/// Backoff intervals for one durable activity retry policy.
584#[derive(Clone, Debug, PartialEq, Eq)]
585pub enum ActivityBackoff {
586    /// Use these intervals between attempts. The server repeats the final
587    /// interval if the retry budget contains more attempts than entries.
588    Explicit(Vec<Duration>),
589    /// Generate one interval for every retry using integer exponential growth.
590    Exponential {
591        initial_interval: Duration,
592        coefficient: u32,
593        maximum_interval: Option<Duration>,
594    },
595}
596
597/// Durable server-side retry policy for one activity execution.
598#[derive(Clone, Debug, Default, PartialEq, Eq)]
599pub struct ActivityRetryPolicy {
600    pub max_attempts: Option<u32>,
601    pub backoff: Option<ActivityBackoff>,
602    pub non_retryable_error_types: Vec<String>,
603}
604
605impl ActivityRetryPolicy {
606    /// Start a policy with a finite attempt budget, including the first attempt.
607    pub fn new(max_attempts: u32) -> Self {
608        Self {
609            max_attempts: Some(max_attempts),
610            ..Self::default()
611        }
612    }
613
614    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
615        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
616        self
617    }
618
619    pub fn exponential_backoff(
620        mut self,
621        initial_interval: Duration,
622        coefficient: u32,
623        maximum_interval: Option<Duration>,
624    ) -> Self {
625        self.backoff = Some(ActivityBackoff::Exponential {
626            initial_interval,
627            coefficient,
628            maximum_interval,
629        });
630        self
631    }
632
633    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
634        self.non_retryable_error_types.push(error_type.into());
635        self
636    }
637
638    pub fn non_retryable_error_types(
639        mut self,
640        error_types: impl IntoIterator<Item = impl Into<String>>,
641    ) -> Self {
642        self.non_retryable_error_types
643            .extend(error_types.into_iter().map(Into::into));
644        self
645    }
646}
647
648/// Options recorded atomically on one deterministic `schedule_activity` command.
649///
650/// Durations are rounded up to whole seconds when encoded, so the server never
651/// receives a shorter timeout or backoff than the caller requested.
652#[derive(Clone, Debug, Default, PartialEq, Eq)]
653pub struct ActivityOptions {
654    pub task_queue: Option<String>,
655    pub retry_policy: Option<ActivityRetryPolicy>,
656    pub start_to_close_timeout: Option<Duration>,
657    pub schedule_to_start_timeout: Option<Duration>,
658    pub schedule_to_close_timeout: Option<Duration>,
659    pub heartbeat_timeout: Option<Duration>,
660}
661
662impl ActivityOptions {
663    pub fn new() -> Self {
664        Self::default()
665    }
666
667    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
668        self.task_queue = Some(task_queue.into());
669        self
670    }
671
672    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
673        self.retry_policy = Some(policy);
674        self
675    }
676
677    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
678        self.start_to_close_timeout = Some(timeout);
679        self
680    }
681
682    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
683        self.schedule_to_start_timeout = Some(timeout);
684        self
685    }
686
687    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
688        self.schedule_to_close_timeout = Some(timeout);
689        self
690    }
691
692    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
693        self.heartbeat_timeout = Some(timeout);
694        self
695    }
696
697    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
698        if self
699            .task_queue
700            .as_deref()
701            .is_some_and(|queue| queue.trim().is_empty())
702        {
703            return Err(ActivityOptionsError::new(
704                ActivityOptionsErrorKind::EmptyTaskQueue,
705                Some("task_queue"),
706                "task_queue must not be empty",
707            ));
708        }
709
710        for (field, value) in [
711            ("start_to_close_timeout", self.start_to_close_timeout),
712            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
713            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
714            ("heartbeat_timeout", self.heartbeat_timeout),
715        ] {
716            if value.is_some_and(|value| value.is_zero()) {
717                return Err(ActivityOptionsError::new(
718                    ActivityOptionsErrorKind::TimeoutNotPositive,
719                    Some(field),
720                    format!("{field} must be positive"),
721                ));
722            }
723        }
724
725        validate_timeout_order(
726            "heartbeat_timeout",
727            self.heartbeat_timeout,
728            "start_to_close_timeout",
729            self.start_to_close_timeout,
730        )?;
731        validate_timeout_order(
732            "start_to_close_timeout",
733            self.start_to_close_timeout,
734            "schedule_to_close_timeout",
735            self.schedule_to_close_timeout,
736        )?;
737        validate_timeout_order(
738            "schedule_to_start_timeout",
739            self.schedule_to_start_timeout,
740            "schedule_to_close_timeout",
741            self.schedule_to_close_timeout,
742        )?;
743
744        Ok(ValidatedActivityOptions {
745            task_queue: self.task_queue.clone(),
746            retry_policy: self
747                .retry_policy
748                .as_ref()
749                .map(validate_activity_retry_policy)
750                .transpose()?,
751            start_to_close_timeout: timeout_seconds(
752                "start_to_close_timeout",
753                self.start_to_close_timeout,
754            )?,
755            schedule_to_start_timeout: timeout_seconds(
756                "schedule_to_start_timeout",
757                self.schedule_to_start_timeout,
758            )?,
759            schedule_to_close_timeout: timeout_seconds(
760                "schedule_to_close_timeout",
761                self.schedule_to_close_timeout,
762            )?,
763            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
764        })
765    }
766}
767
768#[derive(Clone, Debug)]
769struct ValidatedActivityOptions {
770    task_queue: Option<String>,
771    retry_policy: Option<Value>,
772    start_to_close_timeout: Option<u64>,
773    schedule_to_start_timeout: Option<u64>,
774    schedule_to_close_timeout: Option<u64>,
775    heartbeat_timeout: Option<u64>,
776}
777
778fn validate_timeout_order(
779    smaller_name: &'static str,
780    smaller: Option<Duration>,
781    larger_name: &'static str,
782    larger: Option<Duration>,
783) -> std::result::Result<(), ActivityOptionsError> {
784    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
785        return Err(ActivityOptionsError::new(
786            ActivityOptionsErrorKind::TimeoutOrder,
787            Some(smaller_name),
788            format!("{smaller_name} must be <= {larger_name}"),
789        ));
790    }
791    Ok(())
792}
793
794fn timeout_seconds(
795    field: &'static str,
796    value: Option<Duration>,
797) -> std::result::Result<Option<u64>, ActivityOptionsError> {
798    value
799        .map(|value| {
800            activity_protocol_seconds(value).ok_or_else(|| {
801                ActivityOptionsError::new(
802                    ActivityOptionsErrorKind::TimeoutOverflow,
803                    Some(field),
804                    format!("{field} is too large for the worker protocol"),
805                )
806            })
807        })
808        .transpose()
809}
810
811fn duration_seconds_ceil(value: Duration) -> Option<u64> {
812    value
813        .as_secs()
814        .checked_add(u64::from(value.subsec_nanos() > 0))
815}
816
817fn activity_protocol_seconds(value: Duration) -> Option<u64> {
818    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
819}
820
821fn validate_activity_retry_policy(
822    policy: &ActivityRetryPolicy,
823) -> std::result::Result<Value, ActivityOptionsError> {
824    if policy.max_attempts.is_none()
825        && policy.backoff.is_none()
826        && policy.non_retryable_error_types.is_empty()
827    {
828        return Err(ActivityOptionsError::new(
829            ActivityOptionsErrorKind::EmptyRetryPolicy,
830            Some("retry_policy"),
831            "retry_policy must configure at least one field",
832        ));
833    }
834    if policy.max_attempts == Some(0) {
835        return Err(ActivityOptionsError::new(
836            ActivityOptionsErrorKind::InvalidMaxAttempts,
837            Some("retry_policy.max_attempts"),
838            "max_attempts must be >= 1",
839        ));
840    }
841    if policy
842        .non_retryable_error_types
843        .iter()
844        .any(|error_type| error_type.trim().is_empty())
845    {
846        return Err(ActivityOptionsError::new(
847            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
848            Some("retry_policy.non_retryable_error_types"),
849            "non_retryable_error_types must not contain empty values",
850        ));
851    }
852
853    let backoff_seconds = match &policy.backoff {
854        None => None,
855        Some(backoff) => {
856            let max_attempts = policy.max_attempts.ok_or_else(|| {
857                ActivityOptionsError::new(
858                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
859                    Some("retry_policy.backoff"),
860                    "backoff requires max_attempts",
861                )
862            })?;
863            let retry_count = max_attempts.saturating_sub(1) as usize;
864            let intervals = match backoff {
865                ActivityBackoff::Explicit(intervals) => {
866                    if intervals.len() > retry_count {
867                        return Err(ActivityOptionsError::new(
868                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
869                            Some("retry_policy.backoff"),
870                            "backoff interval count must not exceed max_attempts - 1",
871                        ));
872                    }
873                    intervals.clone()
874                }
875                ActivityBackoff::Exponential {
876                    initial_interval,
877                    coefficient,
878                    maximum_interval,
879                } => {
880                    if *coefficient < 1 {
881                        return Err(ActivityOptionsError::new(
882                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
883                            Some("retry_policy.backoff.coefficient"),
884                            "backoff coefficient must be >= 1",
885                        ));
886                    }
887                    if retry_count > 10_000 {
888                        return Err(ActivityOptionsError::new(
889                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
890                            Some("retry_policy.max_attempts"),
891                            "generated backoff supports at most 10000 retry intervals",
892                        ));
893                    }
894                    let mut current = *initial_interval;
895                    let mut intervals = Vec::with_capacity(retry_count);
896                    for _ in 0..retry_count {
897                        let interval = maximum_interval
898                            .map(|maximum| current.min(maximum))
899                            .unwrap_or(current);
900                        intervals.push(interval);
901                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
902                            break;
903                        }
904                        current = current.checked_mul(*coefficient).ok_or_else(|| {
905                            ActivityOptionsError::new(
906                                ActivityOptionsErrorKind::BackoffOverflow,
907                                Some("retry_policy.backoff"),
908                                "generated backoff interval overflowed",
909                            )
910                        })?;
911                    }
912                    intervals
913                }
914            };
915            Some(
916                intervals
917                    .into_iter()
918                    .map(|interval| {
919                        activity_protocol_seconds(interval).ok_or_else(|| {
920                            ActivityOptionsError::new(
921                                ActivityOptionsErrorKind::BackoffOverflow,
922                                Some("retry_policy.backoff"),
923                                "backoff interval is too large for the worker protocol",
924                            )
925                        })
926                    })
927                    .collect::<std::result::Result<Vec<_>, _>>()?,
928            )
929        }
930    };
931
932    let mut encoded = serde_json::Map::new();
933    if let Some(max_attempts) = policy.max_attempts {
934        encoded.insert("max_attempts".to_string(), json!(max_attempts));
935    }
936    if let Some(backoff_seconds) = backoff_seconds {
937        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
938    }
939    if !policy.non_retryable_error_types.is_empty() {
940        let mut canonical_error_types = Vec::new();
941        for error_type in policy
942            .non_retryable_error_types
943            .iter()
944            .map(|error_type| error_type.trim())
945        {
946            if !canonical_error_types.contains(&error_type) {
947                canonical_error_types.push(error_type);
948            }
949        }
950        encoded.insert(
951            "non_retryable_error_types".to_string(),
952            json!(canonical_error_types),
953        );
954    }
955    Ok(Value::Object(encoded))
956}
957
958/// A stable, machine-readable failure raised when workflow code no longer
959/// reconstructs the durable command stream recorded in history.
960#[derive(Clone, Debug, Error)]
961#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
962pub struct ReplayFailure {
963    pub reason: String,
964    pub sequence: Option<u64>,
965    pub expected: Option<String>,
966    pub actual: Option<String>,
967    pub message: String,
968}
969
970impl ReplayFailure {
971    fn new(
972        reason: impl Into<String>,
973        sequence: Option<u64>,
974        expected: Option<String>,
975        actual: Option<String>,
976        message: impl Into<String>,
977    ) -> Self {
978        Self {
979            reason: reason.into(),
980            sequence,
981            expected,
982            actual,
983            message: message.into(),
984        }
985    }
986}
987
988/// A stable, machine-readable workflow query or query-task settlement failure.
989#[derive(Clone, Debug, Error)]
990#[error("query failed ({reason}, HTTP {status}): {message}")]
991pub struct QueryFailure {
992    pub status: u16,
993    pub reason: String,
994    pub message: String,
995    pub body: Value,
996}
997
998/// A stable failure returned when a server rejects an SDK protocol version.
999#[derive(Clone, Debug, Error)]
1000#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1001pub struct ProtocolFailure {
1002    pub status: u16,
1003    pub reason: String,
1004    pub message: String,
1005    pub supported_version: Option<String>,
1006    pub requested_version: Option<String>,
1007    pub body: Value,
1008}
1009
1010#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1011pub struct PayloadEnvelope {
1012    pub codec: String,
1013    pub blob: String,
1014}
1015
1016impl PayloadEnvelope {
1017    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1018        encode_payload(value, DEFAULT_CODEC)
1019    }
1020
1021    pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1022        encode_payload(value, JSON_CODEC)
1023    }
1024
1025    /// Encode an explicit typed value, including the bytes branch that JSON
1026    /// serialization cannot represent.
1027    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1028        encode_avro_value(value)
1029    }
1030}
1031
1032/// Native adapter for the fixed language-neutral Avro Value schema.
1033#[derive(Clone, Debug, PartialEq)]
1034pub enum AvroValue {
1035    Null,
1036    Boolean(bool),
1037    Long(i64),
1038    Double(f64),
1039    Bytes(Vec<u8>),
1040    String(String),
1041    Array(Vec<AvroValue>),
1042    Map(BTreeMap<String, AvroValue>),
1043}
1044
1045impl AvroValue {
1046    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1047        Self::from_serde_value(
1048            serde_value::to_value(value).map_err(|error| {
1049                Error::Codec(format!("could not adapt value for Avro: {error}"))
1050            })?,
1051        )
1052    }
1053
1054    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1055        use serde_value::Value as SerdeValue;
1056
1057        match value {
1058            SerdeValue::Unit => Ok(Self::Null),
1059            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1060            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1061            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1062            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1063            SerdeValue::I64(value) => Ok(Self::Long(value)),
1064            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1065            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1066            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1067            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1068                Error::Codec(
1069                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1070                        .to_string(),
1071                )
1072            }),
1073            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1074            SerdeValue::F64(value) => Self::finite_double(value),
1075            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1076            SerdeValue::String(value) => Ok(Self::String(value)),
1077            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1078            SerdeValue::Option(None) => Ok(Self::Null),
1079            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1080                Self::from_serde_value(*value)
1081            }
1082            SerdeValue::Seq(values) => values
1083                .into_iter()
1084                .map(Self::from_serde_value)
1085                .collect::<Result<Vec<_>>>()
1086                .map(Self::Array),
1087            SerdeValue::Map(values) => values
1088                .into_iter()
1089                .map(|(key, value)| {
1090                    let SerdeValue::String(key) = key else {
1091                        return Err(Error::Codec(
1092                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1093                        ));
1094                    };
1095
1096                    Ok((key, Self::from_serde_value(value)?))
1097                })
1098                .collect::<Result<BTreeMap<_, _>>>()
1099                .map(Self::Map),
1100        }
1101    }
1102
1103    fn finite_double(value: f64) -> Result<Self> {
1104        if !value.is_finite() {
1105            return Err(Error::Codec(
1106                "non_finite_float: Avro Value doubles must be finite".to_string(),
1107            ));
1108        }
1109
1110        Ok(Self::Double(value))
1111    }
1112
1113    fn into_json(self) -> Result<Value> {
1114        match self {
1115            Self::Null => Ok(Value::Null),
1116            Self::Boolean(value) => Ok(Value::Bool(value)),
1117            Self::Long(value) => Ok(Value::Number(value.into())),
1118            Self::Double(value) => serde_json::Number::from_f64(value)
1119                .map(Value::Number)
1120                .ok_or_else(|| {
1121                    Error::Codec(
1122                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1123                    )
1124                }),
1125            Self::Bytes(value) => Ok(json!({
1126                "$type": "bytes",
1127                "base64": BASE64.encode(value),
1128            })),
1129            Self::String(value) => Ok(Value::String(value)),
1130            Self::Array(values) => values
1131                .into_iter()
1132                .map(Self::into_json)
1133                .collect::<Result<Vec<_>>>()
1134                .map(Value::Array),
1135            Self::Map(values) => values
1136                .into_iter()
1137                .map(|(key, value)| Ok((key, value.into_json()?)))
1138                .collect::<Result<serde_json::Map<_, _>>>()
1139                .map(Value::Object),
1140        }
1141    }
1142
1143    fn into_serde_value(self) -> serde_value::Value {
1144        use serde_value::Value as SerdeValue;
1145
1146        match self {
1147            Self::Null => SerdeValue::Unit,
1148            Self::Boolean(value) => SerdeValue::Bool(value),
1149            Self::Long(value) => SerdeValue::I64(value),
1150            Self::Double(value) => SerdeValue::F64(value),
1151            Self::Bytes(value) => SerdeValue::Bytes(value),
1152            Self::String(value) => SerdeValue::String(value),
1153            Self::Array(values) => {
1154                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1155            }
1156            Self::Map(values) => SerdeValue::Map(
1157                values
1158                    .into_iter()
1159                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1160                    .collect(),
1161            ),
1162        }
1163    }
1164
1165    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1166        self.into_serde_value().deserialize_into().map_err(|error| {
1167            Error::Codec(format!(
1168                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1169            ))
1170        })
1171    }
1172}
1173
1174impl Serialize for AvroValue {
1175    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1176    where
1177        S: Serializer,
1178    {
1179        match self {
1180            Self::Null => serializer.serialize_unit(),
1181            Self::Boolean(value) => serializer.serialize_bool(*value),
1182            Self::Long(value) => serializer.serialize_i64(*value),
1183            Self::Double(value) => serializer.serialize_f64(*value),
1184            Self::Bytes(value) => serializer.serialize_bytes(value),
1185            Self::String(value) => serializer.serialize_str(value),
1186            Self::Array(values) => {
1187                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1188                for value in values {
1189                    sequence.serialize_element(value)?;
1190                }
1191                sequence.end()
1192            }
1193            Self::Map(values) => {
1194                let mut map = serializer.serialize_map(Some(values.len()))?;
1195                for (key, value) in values {
1196                    map.serialize_entry(key, value)?;
1197                }
1198                map.end()
1199            }
1200        }
1201    }
1202}
1203
1204pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1205    let datum = avro_value_to_datum(value)?;
1206    let datum = to_avro_datum(avro_value_schema()?, datum)
1207        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1208    let mut bytes = Vec::with_capacity(datum.len() + 10);
1209    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1210    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1211    bytes.extend_from_slice(&datum);
1212    Ok(PayloadEnvelope {
1213        codec: DEFAULT_CODEC.to_string(),
1214        blob: BASE64.encode(bytes),
1215    })
1216}
1217
1218pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1219    if envelope.codec != DEFAULT_CODEC {
1220        return Err(Error::Codec(format!(
1221            "unsupported payload codec {:?}",
1222            envelope.codec
1223        )));
1224    }
1225    decode_avro_value_blob(&envelope.blob)
1226}
1227
1228pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1229    let blob = match codec {
1230        JSON_CODEC => serde_json::to_string(value)?,
1231        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1232        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1233    };
1234
1235    Ok(PayloadEnvelope {
1236        codec: codec.to_string(),
1237        blob,
1238    })
1239}
1240
1241pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1242    match envelope.codec.as_str() {
1243        JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1244        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1245        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1246    }
1247}
1248
1249#[cfg(test)]
1250fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1251    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1252}
1253
1254fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1255    if value.is_null() {
1256        return Ok(Value::Null);
1257    }
1258
1259    if let Some(object) = value.as_object() {
1260        if let (Some(codec), Some(blob)) = (
1261            object.get("codec").and_then(Value::as_str),
1262            object.get("blob").and_then(Value::as_str),
1263        ) {
1264            return decode_blob(blob, codec);
1265        }
1266    }
1267
1268    if let Some(blob) = value.as_str() {
1269        return decode_blob(blob, fallback_codec);
1270    }
1271
1272    Ok(value.clone())
1273}
1274
1275fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1276    let envelope = match codec {
1277        DEFAULT_CODEC => encode_avro_value(value)?,
1278        JSON_CODEC => PayloadEnvelope {
1279            codec: JSON_CODEC.to_string(),
1280            blob: serde_json::to_string(&value.clone().into_json()?)?,
1281        },
1282        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1283    };
1284    Ok(serde_json::to_value(envelope)?)
1285}
1286
1287fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1288    if value.is_null() {
1289        return Ok(AvroValue::Null);
1290    }
1291
1292    if let Some(object) = value.as_object() {
1293        if let (Some(codec), Some(blob)) = (
1294            object.get("codec").and_then(Value::as_str),
1295            object.get("blob").and_then(Value::as_str),
1296        ) {
1297            return match codec {
1298                DEFAULT_CODEC => decode_avro_value_blob(blob),
1299                JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1300                other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1301            };
1302        }
1303    }
1304
1305    if let Some(blob) = value.as_str() {
1306        return match fallback_codec {
1307            DEFAULT_CODEC => decode_avro_value_blob(blob),
1308            JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1309            other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1310        };
1311    }
1312
1313    AvroValue::from_serialize(value)
1314}
1315
1316fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1317    match value {
1318        AvroValue::Null => AvroValue::Array(Vec::new()),
1319        AvroValue::Array(_) => value,
1320        other => AvroValue::Array(vec![other]),
1321    }
1322}
1323
1324fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1325    match codec {
1326        JSON_CODEC => Ok(serde_json::from_str(blob)?),
1327        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1328        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1329    }
1330}
1331
1332fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1333    let bytes = BASE64.decode(blob).map_err(|err| {
1334        Error::Codec(format!(
1335            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1336        ))
1337    })?;
1338
1339    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1340        return Err(Error::Codec(
1341            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1342        ));
1343    }
1344
1345    let fingerprint: [u8; 8] = bytes[2..10]
1346        .try_into()
1347        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1348    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1349        return Err(Error::Codec(format!(
1350            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1351            fingerprint
1352                .iter()
1353                .map(|byte| format!("{byte:02x}"))
1354                .collect::<String>()
1355        )));
1356    }
1357
1358    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1359    // The current fingerprint selects the current immutable schema, so reader
1360    // resolution would only re-walk the same recursive union. Future retained
1361    // writer fingerprints supply a distinct reader schema in this branch.
1362    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1363    if datum_reader.truncated {
1364        return Err(Error::Codec(
1365            "invalid_payload_framing: truncated Avro Value datum".to_string(),
1366        ));
1367    }
1368    let datum = datum.map_err(|err| {
1369        Error::Codec(format!(
1370            "invalid_payload_framing: malformed Avro Value datum: {err}"
1371        ))
1372    })?;
1373    if datum_reader.remaining() != 0 {
1374        return Err(Error::Codec(format!(
1375            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1376            datum_reader.remaining()
1377        )));
1378    }
1379    avro_value_from_datum(datum)
1380}
1381
1382struct StrictAvroDatumReader<'a> {
1383    bytes: &'a [u8],
1384    offset: usize,
1385    truncated: bool,
1386}
1387
1388impl<'a> StrictAvroDatumReader<'a> {
1389    fn new(bytes: &'a [u8]) -> Self {
1390        Self {
1391            bytes,
1392            offset: 0,
1393            truncated: false,
1394        }
1395    }
1396
1397    fn remaining(&self) -> usize {
1398        self.bytes.len() - self.offset
1399    }
1400}
1401
1402impl Read for StrictAvroDatumReader<'_> {
1403    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1404        let count = buffer.len().min(self.remaining());
1405        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1406        self.offset += count;
1407        if count < buffer.len() {
1408            self.truncated = true;
1409        }
1410
1411        Ok(count)
1412    }
1413}
1414
1415fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1416    let branch = match value {
1417        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1418        AvroValue::Boolean(value) => AvroDatum::Union(
1419            1,
1420            Box::new(AvroDatum::Record(vec![(
1421                "boolean".to_string(),
1422                AvroDatum::Boolean(*value),
1423            )])),
1424        ),
1425        AvroValue::Long(value) => AvroDatum::Union(
1426            2,
1427            Box::new(AvroDatum::Record(vec![(
1428                "long".to_string(),
1429                AvroDatum::Long(*value),
1430            )])),
1431        ),
1432        AvroValue::Double(value) => {
1433            if !value.is_finite() {
1434                return Err(Error::Codec(
1435                    "non_finite_float: Avro Value doubles must be finite".to_string(),
1436                ));
1437            }
1438            AvroDatum::Union(
1439                3,
1440                Box::new(AvroDatum::Record(vec![(
1441                    "double".to_string(),
1442                    AvroDatum::Double(*value),
1443                )])),
1444            )
1445        }
1446        AvroValue::Bytes(value) => AvroDatum::Union(
1447            4,
1448            Box::new(AvroDatum::Record(vec![(
1449                "bytes".to_string(),
1450                AvroDatum::Bytes(value.clone()),
1451            )])),
1452        ),
1453        AvroValue::String(value) => AvroDatum::Union(
1454            5,
1455            Box::new(AvroDatum::Record(vec![(
1456                "string".to_string(),
1457                AvroDatum::String(value.clone()),
1458            )])),
1459        ),
1460        AvroValue::Array(values) => AvroDatum::Union(
1461            6,
1462            Box::new(AvroDatum::Record(vec![(
1463                "items".to_string(),
1464                AvroDatum::Array(
1465                    values
1466                        .iter()
1467                        .map(avro_value_to_datum)
1468                        .collect::<Result<Vec<_>>>()?,
1469                ),
1470            )])),
1471        ),
1472        AvroValue::Map(values) => AvroDatum::Union(
1473            7,
1474            Box::new(AvroDatum::Record(vec![(
1475                "entries".to_string(),
1476                AvroDatum::Map(
1477                    values
1478                        .iter()
1479                        .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1480                        .collect::<Result<HashMap<_, _>>>()?,
1481                ),
1482            )])),
1483        ),
1484    };
1485    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1486}
1487
1488fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1489    let AvroDatum::Record(mut outer) = datum else {
1490        return Err(Error::Codec(
1491            "invalid_payload_framing: datum is not a Value record".to_string(),
1492        ));
1493    };
1494    let (_, branch) = outer
1495        .pop()
1496        .filter(|(name, _)| name == "value")
1497        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1498    let AvroDatum::Union(_, branch) = branch else {
1499        return Err(Error::Codec(
1500            "invalid_payload_framing: invalid Value union".to_string(),
1501        ));
1502    };
1503    match *branch {
1504        AvroDatum::Null => Ok(AvroValue::Null),
1505        AvroDatum::Record(mut fields) => {
1506            let (name, value) = fields.pop().ok_or_else(|| {
1507                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1508            })?;
1509            match (name.as_str(), value) {
1510                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1511                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1512                ("double", AvroDatum::Double(value)) if value.is_finite() => {
1513                    Ok(AvroValue::Double(value))
1514                }
1515                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1516                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1517                ("items", AvroDatum::Array(values)) => values
1518                    .into_iter()
1519                    .map(avro_value_from_datum)
1520                    .collect::<Result<Vec<_>>>()
1521                    .map(AvroValue::Array),
1522                ("entries", AvroDatum::Map(values)) => values
1523                    .into_iter()
1524                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1525                    .collect::<Result<BTreeMap<_, _>>>()
1526                    .map(AvroValue::Map),
1527                _ => Err(Error::Codec(
1528                    "invalid_payload_framing: unknown Value branch".to_string(),
1529                )),
1530            }
1531        }
1532        _ => Err(Error::Codec(
1533            "invalid_payload_framing: invalid Value branch".to_string(),
1534        )),
1535    }
1536}
1537
1538fn avro_value_schema() -> Result<&'static Schema> {
1539    match AVRO_VALUE_SCHEMA.get_or_init(|| {
1540        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1541            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1542    }) {
1543        Ok(schema) => Ok(schema),
1544        Err(message) => Err(Error::Codec(message.clone())),
1545    }
1546}
1547
1548#[derive(Clone, Debug)]
1549pub struct Client {
1550    http: reqwest::Client,
1551    base_url: String,
1552    token: Option<String>,
1553    control_token: Option<String>,
1554    worker_token: Option<String>,
1555    namespace: String,
1556}
1557
1558impl Client {
1559    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1560        Self::builder(base_url).build()
1561    }
1562
1563    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1564        ClientBuilder {
1565            base_url: base_url.into(),
1566            token: None,
1567            control_token: None,
1568            worker_token: None,
1569            namespace: "default".to_string(),
1570            timeout: Duration::from_secs(60),
1571        }
1572    }
1573
1574    pub async fn health(&self) -> Result<Value> {
1575        self.request_json(
1576            reqwest::Method::GET,
1577            "/health",
1578            RequestProtocol::ControlPlane,
1579            Option::<&Value>::None,
1580        )
1581        .await
1582    }
1583
1584    pub async fn cluster_info(&self) -> Result<Value> {
1585        self.request_json(
1586            reqwest::Method::GET,
1587            "/cluster/info",
1588            RequestProtocol::ControlPlane,
1589            Option::<&Value>::None,
1590        )
1591        .await
1592    }
1593
1594    pub async fn start_workflow<T: Serialize>(
1595        &self,
1596        workflow_type: &str,
1597        task_queue: &str,
1598        workflow_id: &str,
1599        input: T,
1600    ) -> Result<WorkflowHandle> {
1601        self.start_workflow_with_options(
1602            workflow_type,
1603            task_queue,
1604            workflow_id,
1605            WorkflowStartOptions::default(),
1606            input,
1607        )
1608        .await
1609    }
1610
1611    /// Start a workflow with explicit server-enforced execution and run
1612    /// deadlines.
1613    pub async fn start_workflow_with_options<T: Serialize>(
1614        &self,
1615        workflow_type: &str,
1616        task_queue: &str,
1617        workflow_id: &str,
1618        options: WorkflowStartOptions,
1619        input: T,
1620    ) -> Result<WorkflowHandle> {
1621        options.validate()?;
1622        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1623        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1624        let body = json!({
1625            "workflow_id": workflow_id,
1626            "workflow_type": workflow_type,
1627            "task_queue": task_queue,
1628            "input": input_envelope,
1629            "execution_timeout_seconds": options.execution_timeout_seconds,
1630            "run_timeout_seconds": options.run_timeout_seconds
1631        });
1632
1633        let data: Value = self
1634            .request_json(
1635                reqwest::Method::POST,
1636                "/workflows",
1637                RequestProtocol::ControlPlane,
1638                Some(&body),
1639            )
1640            .await?;
1641
1642        Ok(WorkflowHandle {
1643            client: self.clone(),
1644            workflow_id: data
1645                .get("workflow_id")
1646                .and_then(Value::as_str)
1647                .unwrap_or(workflow_id)
1648                .to_string(),
1649            run_id: data
1650                .get("run_id")
1651                .and_then(Value::as_str)
1652                .map(str::to_string),
1653            workflow_type: data
1654                .get("workflow_type")
1655                .and_then(Value::as_str)
1656                .unwrap_or(workflow_type)
1657                .to_string(),
1658        })
1659    }
1660
1661    pub async fn signal_workflow<T: Serialize>(
1662        &self,
1663        workflow_id: &str,
1664        signal_name: &str,
1665        input: T,
1666    ) -> Result<Value> {
1667        self.signal_workflow_target(workflow_id, None, signal_name, input)
1668            .await
1669    }
1670
1671    /// Signal only if `run_id` is still the current run for this instance.
1672    pub async fn signal_workflow_run<T: Serialize>(
1673        &self,
1674        workflow_id: &str,
1675        run_id: &str,
1676        signal_name: &str,
1677        input: T,
1678    ) -> Result<Value> {
1679        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1680            .await
1681    }
1682
1683    async fn signal_workflow_target<T: Serialize>(
1684        &self,
1685        workflow_id: &str,
1686        run_id: Option<&str>,
1687        signal_name: &str,
1688        input: T,
1689    ) -> Result<Value> {
1690        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1691        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1692        let body = json!({
1693            "input": input_envelope
1694        });
1695        let path = match run_id {
1696            Some(run_id) => {
1697                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1698            }
1699            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1700        };
1701        self.request_json(
1702            reqwest::Method::POST,
1703            &path,
1704            RequestProtocol::ControlPlane,
1705            Some(&body),
1706        )
1707        .await
1708    }
1709
1710    /// Request cooperative cancellation of the current run for an instance.
1711    pub async fn cancel_workflow(
1712        &self,
1713        workflow_id: &str,
1714        options: WorkflowCommandOptions,
1715    ) -> Result<WorkflowCommandResult> {
1716        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1717            .await
1718    }
1719
1720    /// Request cooperative cancellation only if `run_id` is still current.
1721    pub async fn cancel_workflow_run(
1722        &self,
1723        workflow_id: &str,
1724        run_id: &str,
1725        options: WorkflowCommandOptions,
1726    ) -> Result<WorkflowCommandResult> {
1727        self.workflow_command(
1728            workflow_id,
1729            Some(run_id),
1730            WorkflowCommandKind::Cancel,
1731            options,
1732        )
1733        .await
1734    }
1735
1736    /// Forcefully terminate the current run for an instance.
1737    pub async fn terminate_workflow(
1738        &self,
1739        workflow_id: &str,
1740        options: WorkflowCommandOptions,
1741    ) -> Result<WorkflowCommandResult> {
1742        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1743            .await
1744    }
1745
1746    /// Forcefully terminate only if `run_id` is still current.
1747    pub async fn terminate_workflow_run(
1748        &self,
1749        workflow_id: &str,
1750        run_id: &str,
1751        options: WorkflowCommandOptions,
1752    ) -> Result<WorkflowCommandResult> {
1753        self.workflow_command(
1754            workflow_id,
1755            Some(run_id),
1756            WorkflowCommandKind::Terminate,
1757            options,
1758        )
1759        .await
1760    }
1761
1762    async fn workflow_command(
1763        &self,
1764        workflow_id: &str,
1765        run_id: Option<&str>,
1766        command: WorkflowCommandKind,
1767        options: WorkflowCommandOptions,
1768    ) -> Result<WorkflowCommandResult> {
1769        let path = match run_id {
1770            Some(run_id) => format!(
1771                "/workflows/{workflow_id}/runs/{run_id}/{}",
1772                command.as_str()
1773            ),
1774            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1775        };
1776        let data = match self
1777            .request_json(
1778                reqwest::Method::POST,
1779                &path,
1780                RequestProtocol::ControlPlane,
1781                Some(&options),
1782            )
1783            .await
1784        {
1785            Ok(data) => data,
1786            Err(Error::Http { status, body }) => {
1787                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1788                    command,
1789                    status,
1790                    body,
1791                    workflow_id,
1792                    run_id,
1793                )));
1794            }
1795            Err(error) => return Err(error),
1796        };
1797
1798        Ok(workflow_command_result(command, data, workflow_id, run_id))
1799    }
1800
1801    /// Execute a named, read-only query against a running or completed workflow.
1802    ///
1803    /// Arguments and results use the platform payload envelope. Server and
1804    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1805    /// reason, HTTP status, and original response body.
1806    pub async fn query_workflow<T: Serialize>(
1807        &self,
1808        workflow_id: &str,
1809        query_name: &str,
1810        input: T,
1811    ) -> Result<Value> {
1812        self.query_workflow_target(workflow_id, None, query_name, input)
1813            .await
1814    }
1815
1816    /// Query only if `run_id` is still current, preventing accidental retargeting.
1817    pub async fn query_workflow_run<T: Serialize>(
1818        &self,
1819        workflow_id: &str,
1820        run_id: &str,
1821        query_name: &str,
1822        input: T,
1823    ) -> Result<Value> {
1824        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1825            .await
1826    }
1827
1828    /// Query a workflow and return the lossless fixed Avro Value result.
1829    pub async fn query_workflow_avro_value<T: Serialize>(
1830        &self,
1831        workflow_id: &str,
1832        query_name: &str,
1833        input: T,
1834    ) -> Result<AvroValue> {
1835        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1836            .await
1837    }
1838
1839    /// Query a selected run and return the lossless fixed Avro Value result.
1840    pub async fn query_workflow_run_avro_value<T: Serialize>(
1841        &self,
1842        workflow_id: &str,
1843        run_id: &str,
1844        query_name: &str,
1845        input: T,
1846    ) -> Result<AvroValue> {
1847        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1848            .await
1849    }
1850
1851    async fn query_workflow_avro_value_target<T: Serialize>(
1852        &self,
1853        workflow_id: &str,
1854        run_id: Option<&str>,
1855        query_name: &str,
1856        input: T,
1857    ) -> Result<AvroValue> {
1858        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1859        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1860        let path = match run_id {
1861            Some(run_id) => {
1862                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1863            }
1864            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1865        };
1866        let response: Value = match self
1867            .request_json(
1868                reqwest::Method::POST,
1869                &path,
1870                RequestProtocol::ControlPlane,
1871                Some(&body),
1872            )
1873            .await
1874        {
1875            Ok(response) => response,
1876            Err(Error::Http { status, body }) => {
1877                return Err(Error::QueryFailed(query_failure(status, body)));
1878            }
1879            Err(error) => return Err(error),
1880        };
1881
1882        let envelope = response
1883            .get("result_envelope")
1884            .filter(|envelope| !envelope.is_null())
1885            .ok_or_else(|| {
1886                Error::Codec(
1887                    "missing_payload_envelope: typed query result requires result_envelope"
1888                        .to_string(),
1889                )
1890            })?;
1891        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1892    }
1893
1894    async fn query_workflow_target<T: Serialize>(
1895        &self,
1896        workflow_id: &str,
1897        run_id: Option<&str>,
1898        query_name: &str,
1899        input: T,
1900    ) -> Result<Value> {
1901        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1902        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1903        let body = json!({
1904            "input": input_envelope
1905        });
1906        let path = match run_id {
1907            Some(run_id) => {
1908                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1909            }
1910            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1911        };
1912        let response: Value = match self
1913            .request_json(
1914                reqwest::Method::POST,
1915                &path,
1916                RequestProtocol::ControlPlane,
1917                Some(&body),
1918            )
1919            .await
1920        {
1921            Ok(response) => response,
1922            Err(Error::Http { status, body }) => {
1923                return Err(Error::QueryFailed(query_failure(status, body)));
1924            }
1925            Err(error) => return Err(error),
1926        };
1927
1928        if let Some(envelope) = response
1929            .get("result_envelope")
1930            .filter(|envelope| !envelope.is_null())
1931        {
1932            return decode_wire_value(envelope, DEFAULT_CODEC);
1933        }
1934
1935        Ok(response.get("result").cloned().unwrap_or(Value::Null))
1936    }
1937
1938    /// Send a synchronous update using fixed Avro Value arguments.
1939    pub async fn update_workflow<T: Serialize>(
1940        &self,
1941        workflow_id: &str,
1942        update_name: &str,
1943        input: T,
1944        request_id: Option<&str>,
1945    ) -> Result<Value> {
1946        let response = self
1947            .update_workflow_response(workflow_id, update_name, input, request_id)
1948            .await?;
1949        if let Some(envelope) = response
1950            .get("result_envelope")
1951            .filter(|envelope| !envelope.is_null())
1952        {
1953            return decode_wire_value(envelope, DEFAULT_CODEC);
1954        }
1955        Ok(response.get("result").cloned().unwrap_or(response))
1956    }
1957
1958    /// Send a synchronous update and retain a bytes-capable Avro result.
1959    pub async fn update_workflow_avro_value<T: Serialize>(
1960        &self,
1961        workflow_id: &str,
1962        update_name: &str,
1963        input: T,
1964        request_id: Option<&str>,
1965    ) -> Result<AvroValue> {
1966        let response = self
1967            .update_workflow_response(workflow_id, update_name, input, request_id)
1968            .await?;
1969        let envelope = response
1970            .get("result_envelope")
1971            .filter(|envelope| !envelope.is_null())
1972            .ok_or_else(|| {
1973                Error::Codec(
1974                    "missing_payload_envelope: typed update result requires result_envelope"
1975                        .to_string(),
1976                )
1977            })?;
1978        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1979    }
1980
1981    async fn update_workflow_response<T: Serialize>(
1982        &self,
1983        workflow_id: &str,
1984        update_name: &str,
1985        input: T,
1986        request_id: Option<&str>,
1987    ) -> Result<Value> {
1988        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1989        let mut body = json!({
1990            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1991            "wait_for": "completed",
1992        });
1993        if let Some(request_id) = request_id {
1994            body["request_id"] = json!(request_id);
1995        }
1996        self.request_json(
1997            reqwest::Method::POST,
1998            &format!("/workflows/{workflow_id}/update/{update_name}"),
1999            RequestProtocol::ControlPlane,
2000            Some(&body),
2001        )
2002        .await
2003    }
2004
2005    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2006        let path = format!("/workflows/{workflow_id}");
2007        let mut data: WorkflowDescription = self
2008            .request_json(
2009                reqwest::Method::GET,
2010                &path,
2011                RequestProtocol::ControlPlane,
2012                Option::<&Value>::None,
2013            )
2014            .await?;
2015        data.decode_payloads()?;
2016        Ok(data)
2017    }
2018
2019    /// Describe one selected run, including historical terminal runs.
2020    pub async fn describe_workflow_run(
2021        &self,
2022        workflow_id: &str,
2023        run_id: &str,
2024    ) -> Result<WorkflowDescription> {
2025        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2026        let mut data: WorkflowDescription = self
2027            .request_json(
2028                reqwest::Method::GET,
2029                &path,
2030                RequestProtocol::ControlPlane,
2031                Option::<&Value>::None,
2032            )
2033            .await?;
2034        data.decode_payloads()?;
2035        Ok(data)
2036    }
2037
2038    pub async fn register_worker(
2039        &self,
2040        worker_id: &str,
2041        task_queue: &str,
2042        supported_workflow_types: Vec<String>,
2043        supported_activity_types: Vec<String>,
2044        max_concurrent_workflow_tasks: usize,
2045        max_concurrent_activity_tasks: usize,
2046    ) -> Result<RegisterWorkerResponse> {
2047        self.register_worker_with_capabilities(
2048            worker_id,
2049            task_queue,
2050            supported_workflow_types,
2051            supported_activity_types,
2052            max_concurrent_workflow_tasks,
2053            max_concurrent_activity_tasks,
2054            Vec::new(),
2055        )
2056        .await
2057    }
2058
2059    /// Register a worker and explicitly advertise additive worker capabilities.
2060    pub async fn register_worker_with_capabilities(
2061        &self,
2062        worker_id: &str,
2063        task_queue: &str,
2064        supported_workflow_types: Vec<String>,
2065        supported_activity_types: Vec<String>,
2066        max_concurrent_workflow_tasks: usize,
2067        max_concurrent_activity_tasks: usize,
2068        capabilities: Vec<String>,
2069    ) -> Result<RegisterWorkerResponse> {
2070        self.register_worker_with_command_contracts(
2071            worker_id,
2072            task_queue,
2073            supported_workflow_types,
2074            supported_activity_types,
2075            max_concurrent_workflow_tasks,
2076            max_concurrent_activity_tasks,
2077            capabilities,
2078            Value::Object(serde_json::Map::new()),
2079        )
2080        .await
2081    }
2082
2083    /// Register a worker and advertise its named query and update handlers.
2084    #[allow(clippy::too_many_arguments)]
2085    pub async fn register_worker_with_command_contracts(
2086        &self,
2087        worker_id: &str,
2088        task_queue: &str,
2089        supported_workflow_types: Vec<String>,
2090        supported_activity_types: Vec<String>,
2091        max_concurrent_workflow_tasks: usize,
2092        max_concurrent_activity_tasks: usize,
2093        capabilities: Vec<String>,
2094        workflow_command_contracts: Value,
2095    ) -> Result<RegisterWorkerResponse> {
2096        let mut body = json!({
2097            "worker_id": worker_id,
2098            "task_queue": task_queue,
2099            "runtime": "rust",
2100            "sdk_version": SDK_VERSION,
2101            "supported_workflow_types": supported_workflow_types,
2102            "supported_activity_types": supported_activity_types,
2103            "capabilities": capabilities,
2104            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2105            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2106        });
2107        if workflow_command_contracts
2108            .as_object()
2109            .is_some_and(|contracts| !contracts.is_empty())
2110        {
2111            body["workflow_command_contracts"] = workflow_command_contracts;
2112        }
2113
2114        self.request_json(
2115            reqwest::Method::POST,
2116            "/worker/register",
2117            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2118            Some(&body),
2119        )
2120        .await
2121    }
2122
2123    /// Gracefully remove one worker's registration through the worker plane.
2124    ///
2125    /// This operation is separate from operator-facing worker management. It
2126    /// uses worker-protocol authentication and returns the server's lease
2127    /// recovery result.
2128    pub async fn deregister_worker_registration(
2129        &self,
2130        worker_id: &str,
2131    ) -> Result<WorkerDeregistrationEnvelope> {
2132        let path = format!(
2133            "/worker/registrations/{}",
2134            percent_encode_path_segment(worker_id)
2135        );
2136        self.request_json(
2137            reqwest::Method::DELETE,
2138            &path,
2139            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2140            Option::<&Value>::None,
2141        )
2142        .await
2143    }
2144
2145    /// Long-poll for an ephemeral, read-only workflow query task.
2146    pub async fn poll_query_task(
2147        &self,
2148        worker_id: &str,
2149        task_queue: &str,
2150        timeout: Duration,
2151    ) -> Result<Option<QueryTask>> {
2152        Ok(self
2153            .poll_query_task_response(worker_id, task_queue, timeout)
2154            .await?
2155            .task)
2156    }
2157
2158    /// Poll a query task while preserving server stop and drain metadata.
2159    pub async fn poll_query_task_response(
2160        &self,
2161        worker_id: &str,
2162        task_queue: &str,
2163        timeout: Duration,
2164    ) -> Result<PollQueryTaskResponse> {
2165        let poll_request_id = unique_request_id("rust-query-poll");
2166        self.poll_query_task_response_with_request_id(
2167            worker_id,
2168            task_queue,
2169            timeout,
2170            &poll_request_id,
2171            1,
2172        )
2173        .await
2174    }
2175
2176    async fn poll_query_task_response_with_request_id(
2177        &self,
2178        worker_id: &str,
2179        task_queue: &str,
2180        timeout: Duration,
2181        poll_request_id: &str,
2182        transport_retries: usize,
2183    ) -> Result<PollQueryTaskResponse> {
2184        let timeout_seconds = long_poll_timeout_seconds(timeout);
2185        let body = json!({
2186            "worker_id": worker_id,
2187            "task_queue": task_queue,
2188            "poll_request_id": poll_request_id,
2189            "timeout_seconds": timeout_seconds,
2190        });
2191        self.poll_request_json(
2192            "/worker/query-tasks/poll",
2193            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2194            &body,
2195            timeout + Duration::from_secs(5),
2196            transport_retries,
2197        )
2198        .await
2199    }
2200
2201    /// Complete a query task without appending workflow history.
2202    pub async fn complete_query_task<T: Serialize>(
2203        &self,
2204        query_task_id: &str,
2205        lease_owner: &str,
2206        query_task_attempt: u64,
2207        result: T,
2208        codec: &str,
2209    ) -> Result<Value> {
2210        let typed_result = AvroValue::from_serialize(&result)?;
2211        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2212        self.complete_query_task_with_envelope(
2213            query_task_id,
2214            lease_owner,
2215            query_task_attempt,
2216            typed_result.into_json()?,
2217            result_envelope,
2218        )
2219        .await
2220    }
2221
2222    async fn complete_query_task_with_envelope(
2223        &self,
2224        query_task_id: &str,
2225        lease_owner: &str,
2226        query_task_attempt: u64,
2227        result: Value,
2228        result_envelope: Value,
2229    ) -> Result<Value> {
2230        let body = json!({
2231            "lease_owner": lease_owner,
2232            "query_task_attempt": query_task_attempt,
2233            "result": result,
2234            "result_envelope": result_envelope,
2235        });
2236        let path = format!("/worker/query-tasks/{query_task_id}/complete");
2237        let response = self
2238            .request_json(
2239                reqwest::Method::POST,
2240                &path,
2241                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2242                Some(&body),
2243            )
2244            .await;
2245        query_task_response(response)
2246    }
2247
2248    /// Report a stable machine-readable query-task failure.
2249    pub async fn fail_query_task(
2250        &self,
2251        query_task_id: &str,
2252        lease_owner: &str,
2253        query_task_attempt: u64,
2254        message: impl Into<String>,
2255        reason: impl Into<String>,
2256        failure_type: impl Into<String>,
2257    ) -> Result<Value> {
2258        let body = json!({
2259            "lease_owner": lease_owner,
2260            "query_task_attempt": query_task_attempt,
2261            "failure": {
2262                "message": message.into(),
2263                "reason": reason.into(),
2264                "type": failure_type.into(),
2265            }
2266        });
2267        let path = format!("/worker/query-tasks/{query_task_id}/fail");
2268        let response = self
2269            .request_json(
2270                reqwest::Method::POST,
2271                &path,
2272                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2273                Some(&body),
2274            )
2275            .await;
2276        query_task_response(response)
2277    }
2278
2279    pub async fn heartbeat_worker(
2280        &self,
2281        worker_id: &str,
2282        workflow_available: usize,
2283        activity_available: usize,
2284    ) -> Result<Value> {
2285        let body = json!({
2286            "worker_id": worker_id,
2287            "task_slots": {
2288                "workflow_available": workflow_available,
2289                "activity_available": activity_available
2290            },
2291            "process_metrics": {
2292                "process_id": std::process::id(),
2293                "process_uptime_seconds": 0
2294            }
2295        });
2296
2297        self.request_json(
2298            reqwest::Method::POST,
2299            "/worker/heartbeat",
2300            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2301            Some(&body),
2302        )
2303        .await
2304    }
2305
2306    pub async fn poll_workflow_task(
2307        &self,
2308        worker_id: &str,
2309        task_queue: &str,
2310        timeout: Duration,
2311    ) -> Result<Option<WorkflowTask>> {
2312        Ok(self
2313            .poll_workflow_task_response(worker_id, task_queue, timeout)
2314            .await?
2315            .task)
2316    }
2317
2318    pub async fn poll_workflow_task_response(
2319        &self,
2320        worker_id: &str,
2321        task_queue: &str,
2322        timeout: Duration,
2323    ) -> Result<PollWorkflowTaskResponse> {
2324        let poll_request_id = unique_request_id("rust-workflow-poll");
2325        self.poll_workflow_task_response_with_request_id(
2326            worker_id,
2327            task_queue,
2328            timeout,
2329            &poll_request_id,
2330            1,
2331        )
2332        .await
2333    }
2334
2335    async fn poll_workflow_task_response_with_request_id(
2336        &self,
2337        worker_id: &str,
2338        task_queue: &str,
2339        timeout: Duration,
2340        poll_request_id: &str,
2341        transport_retries: usize,
2342    ) -> Result<PollWorkflowTaskResponse> {
2343        let body = json!({
2344            "worker_id": worker_id,
2345            "task_queue": task_queue,
2346            "poll_request_id": poll_request_id,
2347            "timeout_seconds": long_poll_timeout_seconds(timeout),
2348        });
2349        let mut data: PollWorkflowTaskResponse = self
2350            .poll_request_json(
2351                "/worker/workflow-tasks/poll",
2352                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2353                &body,
2354                timeout + Duration::from_secs(5),
2355                transport_retries,
2356            )
2357            .await?;
2358
2359        if let Some(task) = data.task.as_mut() {
2360            self.fetch_remaining_workflow_history(worker_id, task)
2361                .await?;
2362        }
2363
2364        Ok(data)
2365    }
2366
2367    async fn fetch_remaining_workflow_history(
2368        &self,
2369        worker_id: &str,
2370        task: &mut WorkflowTask,
2371    ) -> Result<()> {
2372        let mut next_token = task.next_history_page_token.clone();
2373
2374        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2375            let lease_owner = task
2376                .lease_owner
2377                .clone()
2378                .unwrap_or_else(|| worker_id.to_string());
2379            let page = self
2380                .workflow_task_history_page(
2381                    &task.task_id,
2382                    &lease_owner,
2383                    task.workflow_task_attempt,
2384                    &token,
2385                )
2386                .await?;
2387
2388            task.append_history_page(page);
2389
2390            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2391                return Err(Error::Codec(
2392                    "workflow history pagination returned the same page token".to_string(),
2393                ));
2394            }
2395
2396            next_token = task.next_history_page_token.clone();
2397        }
2398
2399        Ok(())
2400    }
2401
2402    async fn workflow_task_history_page(
2403        &self,
2404        task_id: &str,
2405        lease_owner: &str,
2406        workflow_task_attempt: u64,
2407        next_history_page_token: &str,
2408    ) -> Result<WorkflowTaskHistoryPage> {
2409        let body = json!({
2410            "lease_owner": lease_owner,
2411            "workflow_task_attempt": workflow_task_attempt,
2412            "next_history_page_token": next_history_page_token
2413        });
2414        let path = format!("/worker/workflow-tasks/{task_id}/history");
2415
2416        self.request_json(
2417            reqwest::Method::POST,
2418            &path,
2419            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2420            Some(&body),
2421        )
2422        .await
2423    }
2424
2425    pub async fn complete_workflow_task(
2426        &self,
2427        task_id: &str,
2428        lease_owner: &str,
2429        workflow_task_attempt: u64,
2430        commands: Vec<Value>,
2431    ) -> Result<Value> {
2432        let body = json!({
2433            "lease_owner": lease_owner,
2434            "workflow_task_attempt": workflow_task_attempt,
2435            "commands": commands
2436        });
2437        let path = format!("/worker/workflow-tasks/{task_id}/complete");
2438        self.request_json(
2439            reqwest::Method::POST,
2440            &path,
2441            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2442            Some(&body),
2443        )
2444        .await
2445    }
2446
2447    pub async fn fail_workflow_task(
2448        &self,
2449        task_id: &str,
2450        lease_owner: &str,
2451        workflow_task_attempt: u64,
2452        message: impl Into<String>,
2453    ) -> Result<Value> {
2454        self.fail_workflow_task_with_type(
2455            task_id,
2456            lease_owner,
2457            workflow_task_attempt,
2458            message,
2459            "RustWorkflowTaskFailure",
2460        )
2461        .await
2462    }
2463
2464    async fn fail_workflow_task_with_type(
2465        &self,
2466        task_id: &str,
2467        lease_owner: &str,
2468        workflow_task_attempt: u64,
2469        message: impl Into<String>,
2470        failure_type: &str,
2471    ) -> Result<Value> {
2472        let body = json!({
2473            "lease_owner": lease_owner,
2474            "workflow_task_attempt": workflow_task_attempt,
2475            "failure": {
2476                "message": message.into(),
2477                "type": failure_type
2478            }
2479        });
2480        let path = format!("/worker/workflow-tasks/{task_id}/fail");
2481        self.request_json(
2482            reqwest::Method::POST,
2483            &path,
2484            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2485            Some(&body),
2486        )
2487        .await
2488    }
2489
2490    pub async fn poll_activity_task(
2491        &self,
2492        worker_id: &str,
2493        task_queue: &str,
2494        timeout: Duration,
2495    ) -> Result<Option<ActivityTask>> {
2496        Ok(self
2497            .poll_activity_task_response(worker_id, task_queue, timeout)
2498            .await?
2499            .task)
2500    }
2501
2502    /// Poll an activity task while preserving server stop and drain metadata.
2503    pub async fn poll_activity_task_response(
2504        &self,
2505        worker_id: &str,
2506        task_queue: &str,
2507        timeout: Duration,
2508    ) -> Result<PollActivityTaskResponse> {
2509        let poll_request_id = unique_request_id("rust-activity-poll");
2510        self.poll_activity_task_response_with_request_id(
2511            worker_id,
2512            task_queue,
2513            timeout,
2514            &poll_request_id,
2515            1,
2516        )
2517        .await
2518    }
2519
2520    async fn poll_activity_task_response_with_request_id(
2521        &self,
2522        worker_id: &str,
2523        task_queue: &str,
2524        timeout: Duration,
2525        poll_request_id: &str,
2526        transport_retries: usize,
2527    ) -> Result<PollActivityTaskResponse> {
2528        let body = json!({
2529            "worker_id": worker_id,
2530            "task_queue": task_queue,
2531            "poll_request_id": poll_request_id,
2532            "timeout_seconds": long_poll_timeout_seconds(timeout),
2533        });
2534        let data: PollActivityTaskResponse = self
2535            .poll_request_json(
2536                "/worker/activity-tasks/poll",
2537                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2538                &body,
2539                timeout + Duration::from_secs(5),
2540                transport_retries,
2541            )
2542            .await?;
2543        Ok(data)
2544    }
2545
2546    pub async fn complete_activity_task<T: Serialize>(
2547        &self,
2548        task_id: &str,
2549        activity_attempt_id: &str,
2550        lease_owner: &str,
2551        result: T,
2552        codec: &str,
2553    ) -> Result<Value> {
2554        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2555        let body = json!({
2556            "activity_attempt_id": activity_attempt_id,
2557            "lease_owner": lease_owner,
2558            "result": result
2559        });
2560        let path = format!("/worker/activity-tasks/{task_id}/complete");
2561        activity_task_response(
2562            self.request_json(
2563                reqwest::Method::POST,
2564                &path,
2565                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2566                Some(&body),
2567            )
2568            .await,
2569            "complete",
2570            task_id,
2571            activity_attempt_id,
2572        )
2573    }
2574
2575    pub async fn fail_activity_task(
2576        &self,
2577        task_id: &str,
2578        activity_attempt_id: &str,
2579        lease_owner: &str,
2580        message: impl Into<String>,
2581        non_retryable: bool,
2582    ) -> Result<Value> {
2583        let body = json!({
2584            "activity_attempt_id": activity_attempt_id,
2585            "lease_owner": lease_owner,
2586            "failure": {
2587                "message": message.into(),
2588                "type": "RustActivityFailure",
2589                "non_retryable": non_retryable
2590            }
2591        });
2592        let path = format!("/worker/activity-tasks/{task_id}/fail");
2593        activity_task_response(
2594            self.request_json(
2595                reqwest::Method::POST,
2596                &path,
2597                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2598                Some(&body),
2599            )
2600            .await,
2601            "fail",
2602            task_id,
2603            activity_attempt_id,
2604        )
2605    }
2606
2607    pub async fn heartbeat_activity_task<T: Serialize>(
2608        &self,
2609        task_id: &str,
2610        activity_attempt_id: &str,
2611        lease_owner: &str,
2612        details: T,
2613    ) -> Result<ActivityHeartbeatResponse> {
2614        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2615        let body = json!({
2616            "activity_attempt_id": activity_attempt_id,
2617            "lease_owner": lease_owner,
2618            "details": details
2619        });
2620        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2621        activity_task_response(
2622            self.request_json(
2623                reqwest::Method::POST,
2624                &path,
2625                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2626                Some(&body),
2627            )
2628            .await,
2629            "heartbeat",
2630            task_id,
2631            activity_attempt_id,
2632        )
2633    }
2634
2635    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2636        &self,
2637        method: reqwest::Method,
2638        path: &str,
2639        protocol: RequestProtocol,
2640        body: Option<&B>,
2641    ) -> Result<T> {
2642        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2643            .await
2644    }
2645
2646    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2647        &self,
2648        method: reqwest::Method,
2649        path: &str,
2650        protocol: RequestProtocol,
2651        body: Option<&B>,
2652        timeout: Duration,
2653    ) -> Result<T> {
2654        let auth_token = self.auth_token(protocol)?;
2655        let mut request = self
2656            .http
2657            .request(method, format!("{}/api{}", self.base_url, path))
2658            .timeout(timeout)
2659            .header(reqwest::header::ACCEPT, "application/json")
2660            .header(reqwest::header::CONTENT_TYPE, "application/json")
2661            .header("X-Namespace", &self.namespace);
2662
2663        match protocol {
2664            RequestProtocol::Worker(version) => {
2665                request = request.header("X-Durable-Workflow-Protocol-Version", version);
2666            }
2667            RequestProtocol::ControlPlane => {
2668                request = request.header(
2669                    "X-Durable-Workflow-Control-Plane-Version",
2670                    CONTROL_PLANE_VERSION,
2671                );
2672            }
2673        }
2674
2675        if let Some(token) = auth_token {
2676            request = request.bearer_auth(token);
2677        }
2678
2679        if let Some(body) = body {
2680            request = request.json(body);
2681        }
2682
2683        let response = request.send().await?;
2684        let status = response.status();
2685        let bytes = response.bytes().await?;
2686
2687        if !status.is_success() {
2688            let body = String::from_utf8_lossy(&bytes).to_string();
2689            if let Some(protocol) = protocol_failure(status, &body) {
2690                return Err(Error::Protocol(protocol));
2691            }
2692            return Err(Error::Http { status, body });
2693        }
2694
2695        if bytes.is_empty() {
2696            return Ok(serde_json::from_value(Value::Null)?);
2697        }
2698
2699        Ok(serde_json::from_slice(&bytes)?)
2700    }
2701
2702    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2703        &self,
2704        path: &str,
2705        protocol: RequestProtocol,
2706        body: &B,
2707        timeout: Duration,
2708        max_retries: usize,
2709    ) -> Result<T> {
2710        let mut retries = 0;
2711
2712        loop {
2713            let response = self
2714                .request_json_with_timeout(
2715                    reqwest::Method::POST,
2716                    path,
2717                    protocol,
2718                    Some(body),
2719                    timeout,
2720                )
2721                .await;
2722
2723            match response {
2724                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2725                response => return worker_poll_response(response),
2726            }
2727        }
2728    }
2729
2730    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2731        match protocol {
2732            RequestProtocol::Worker(_) => {
2733                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2734                    return Ok(Some(token));
2735                }
2736                if self.control_token.is_some() {
2737                    return Err(Error::MissingRoleCredentials {
2738                        role: "worker",
2739                        opposite_role: "control",
2740                    });
2741                }
2742                Ok(None)
2743            }
2744            RequestProtocol::ControlPlane => {
2745                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2746                    return Ok(Some(token));
2747                }
2748                if self.worker_token.is_some() {
2749                    return Err(Error::MissingRoleCredentials {
2750                        role: "control",
2751                        opposite_role: "worker",
2752                    });
2753                }
2754                Ok(None)
2755            }
2756        }
2757    }
2758}
2759
2760fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2761    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2762    let reason = body
2763        .get("reason")
2764        .and_then(Value::as_str)
2765        .unwrap_or("query_rejected")
2766        .to_string();
2767    let message = body
2768        .get("message")
2769        .or_else(|| body.get("error"))
2770        .and_then(Value::as_str)
2771        .unwrap_or("workflow query was rejected")
2772        .to_string();
2773
2774    QueryFailure {
2775        status: status.as_u16(),
2776        reason,
2777        message,
2778        body,
2779    }
2780}
2781
2782fn workflow_command_result(
2783    command: WorkflowCommandKind,
2784    data: Value,
2785    workflow_id: &str,
2786    run_id: Option<&str>,
2787) -> WorkflowCommandResult {
2788    WorkflowCommandResult {
2789        command,
2790        workflow_id: data
2791            .get("workflow_id")
2792            .and_then(Value::as_str)
2793            .unwrap_or(workflow_id)
2794            .to_string(),
2795        run_id: data
2796            .get("run_id")
2797            .and_then(Value::as_str)
2798            .or(run_id)
2799            .map(str::to_string),
2800        outcome: data
2801            .get("outcome")
2802            .and_then(Value::as_str)
2803            .map(str::to_string),
2804        reason: data
2805            .get("reason")
2806            .and_then(Value::as_str)
2807            .map(str::to_string),
2808        command_status: data
2809            .get("command_status")
2810            .and_then(Value::as_str)
2811            .map(str::to_string),
2812        raw: data,
2813    }
2814}
2815
2816fn workflow_command_rejection(
2817    command: WorkflowCommandKind,
2818    status: reqwest::StatusCode,
2819    raw_body: String,
2820    workflow_id: &str,
2821    run_id: Option<&str>,
2822) -> WorkflowCommandRejection {
2823    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2824    WorkflowCommandRejection {
2825        command,
2826        status: status.as_u16(),
2827        reason: body
2828            .get("reason")
2829            .and_then(Value::as_str)
2830            .unwrap_or("workflow_command_rejected")
2831            .to_string(),
2832        message: body
2833            .get("message")
2834            .or_else(|| body.get("error"))
2835            .and_then(Value::as_str)
2836            .unwrap_or("workflow lifecycle command was rejected")
2837            .to_string(),
2838        workflow_id: body
2839            .get("workflow_id")
2840            .and_then(Value::as_str)
2841            .unwrap_or(workflow_id)
2842            .to_string(),
2843        run_id: body
2844            .get("run_id")
2845            .and_then(Value::as_str)
2846            .or(run_id)
2847            .map(str::to_string),
2848        target_scope: body
2849            .get("target_scope")
2850            .and_then(Value::as_str)
2851            .map(str::to_string),
2852        body,
2853    }
2854}
2855
2856fn query_task_response(response: Result<Value>) -> Result<Value> {
2857    match response {
2858        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2859        response => response,
2860    }
2861}
2862
2863fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2864    match response {
2865        Err(Error::Http { status, body })
2866            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2867        {
2868            Ok(serde_json::from_str(&body)?)
2869        }
2870        response => response,
2871    }
2872}
2873
2874fn worker_poll_body_is_stop(body: &str) -> bool {
2875    serde_json::from_str::<Value>(body)
2876        .ok()
2877        .is_some_and(|body| {
2878            worker_poll_is_stop(
2879                body.get("poll_status").and_then(Value::as_str),
2880                body.get("reason").and_then(Value::as_str),
2881            )
2882        })
2883}
2884
2885fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2886    matches!(poll_status, Some("draining" | "stopped"))
2887        || matches!(reason, Some("worker_draining" | "worker_stopped"))
2888}
2889
2890fn query_task_rejection_is_final(error: &Error) -> bool {
2891    matches!(
2892        error,
2893        Error::QueryFailed(failure)
2894            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2895    )
2896}
2897
2898fn activity_task_response<T>(
2899    response: Result<T>,
2900    operation: &str,
2901    task_id: &str,
2902    activity_attempt_id: &str,
2903) -> Result<T> {
2904    match response {
2905        Err(Error::Http { status, body }) => {
2906            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2907            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2908                operation: operation.to_string(),
2909                status: status.as_u16(),
2910                reason: body
2911                    .get("reason")
2912                    .and_then(Value::as_str)
2913                    .unwrap_or("activity_task_rejected")
2914                    .to_string(),
2915                task_id: body
2916                    .get("task_id")
2917                    .and_then(Value::as_str)
2918                    .unwrap_or(task_id)
2919                    .to_string(),
2920                activity_attempt_id: body
2921                    .get("activity_attempt_id")
2922                    .and_then(Value::as_str)
2923                    .unwrap_or(activity_attempt_id)
2924                    .to_string(),
2925                cancel_requested: body
2926                    .get("cancel_requested")
2927                    .and_then(Value::as_bool)
2928                    .unwrap_or(false),
2929                can_continue: body.get("can_continue").and_then(Value::as_bool),
2930                run_closed_reason: body
2931                    .get("run_closed_reason")
2932                    .and_then(Value::as_str)
2933                    .map(str::to_string),
2934                body,
2935            }))
2936        }
2937        response => response,
2938    }
2939}
2940
2941fn activity_task_rejection_is_final(error: &Error) -> bool {
2942    matches!(
2943        error,
2944        Error::ActivityTaskRejected(rejection)
2945            if matches!(
2946                rejection.reason.as_str(),
2947                "run_cancelled"
2948                    | "run_terminated"
2949                    | "attempt_closed"
2950                    | "stale_attempt"
2951                    | "activity_cancelled"
2952                    | "task_cancelled"
2953                    | "run_closed"
2954                    | "activity_not_running"
2955                    | "attempt_not_found"
2956            )
2957    )
2958}
2959
2960fn workflow_task_completion_is_terminal_timeout(
2961    error: &Error,
2962    task_id: &str,
2963    workflow_task_attempt: u64,
2964    run_id: Option<&str>,
2965) -> bool {
2966    let Error::Http { status, body } = error else {
2967        return false;
2968    };
2969    if *status != reqwest::StatusCode::CONFLICT {
2970        return false;
2971    }
2972
2973    let Some(run_id) = run_id else {
2974        return false;
2975    };
2976    let Ok(body) = serde_json::from_str::<Value>(body) else {
2977        return false;
2978    };
2979
2980    body.get("recorded").and_then(Value::as_bool) == Some(false)
2981        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
2982        && body.get("run_status").and_then(Value::as_str) == Some("failed")
2983        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
2984        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
2985        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
2986}
2987
2988fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2989    let body: Value = serde_json::from_str(raw_body).ok()?;
2990    let reason = body.get("reason")?.as_str()?;
2991    if !matches!(
2992        reason,
2993        "missing_protocol_version"
2994            | "unsupported_protocol_version"
2995            | "missing_control_plane_version"
2996            | "unsupported_control_plane_version"
2997    ) {
2998        return None;
2999    }
3000
3001    Some(ProtocolFailure {
3002        status: status.as_u16(),
3003        reason: reason.to_string(),
3004        message: body
3005            .get("message")
3006            .or_else(|| body.get("error"))
3007            .and_then(Value::as_str)
3008            .unwrap_or("protocol version rejected")
3009            .to_string(),
3010        supported_version: body
3011            .get("supported_version")
3012            .and_then(Value::as_str)
3013            .map(str::to_string),
3014        requested_version: body
3015            .get("requested_version")
3016            .and_then(Value::as_str)
3017            .map(str::to_string),
3018        body,
3019    })
3020}
3021
3022fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3023    timeout
3024        .as_secs()
3025        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3026        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3027}
3028
3029fn worker_operation_is_retryable(error: &Error) -> bool {
3030    match error {
3031        Error::Transport(error) => {
3032            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3033        }
3034        Error::Http { status, .. } => {
3035            matches!(
3036                *status,
3037                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3038            ) || status.is_server_error()
3039        }
3040        _ => false,
3041    }
3042}
3043
3044fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3045    let exponent = retry.saturating_sub(1).min(31) as u32;
3046    policy
3047        .initial_backoff
3048        .saturating_mul(1_u32 << exponent)
3049        .min(policy.max_backoff)
3050}
3051
3052#[derive(Debug)]
3053pub struct ClientBuilder {
3054    base_url: String,
3055    token: Option<String>,
3056    control_token: Option<String>,
3057    worker_token: Option<String>,
3058    namespace: String,
3059    timeout: Duration,
3060}
3061
3062impl ClientBuilder {
3063    pub fn token(mut self, token: Option<String>) -> Self {
3064        self.token = token;
3065        self
3066    }
3067
3068    pub fn control_token(mut self, token: Option<String>) -> Self {
3069        self.control_token = token;
3070        self
3071    }
3072
3073    pub fn worker_token(mut self, token: Option<String>) -> Self {
3074        self.worker_token = token;
3075        self
3076    }
3077
3078    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3079        self.namespace = namespace.into();
3080        self
3081    }
3082
3083    pub fn timeout(mut self, timeout: Duration) -> Self {
3084        self.timeout = timeout;
3085        self
3086    }
3087
3088    pub fn build(self) -> Result<Client> {
3089        let base_url = self.base_url.trim_end_matches('/').to_string();
3090        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3091            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3092            .unwrap_or_else(|_| base_url.ends_with("/api"));
3093
3094        if has_sdk_api_suffix {
3095            return Err(Error::InvalidBaseUrl);
3096        }
3097
3098        Ok(Client {
3099            http: reqwest::Client::builder().timeout(self.timeout).build()?,
3100            base_url,
3101            token: self.token,
3102            control_token: self.control_token,
3103            worker_token: self.worker_token,
3104            namespace: self.namespace,
3105        })
3106    }
3107}
3108
3109#[derive(Clone, Debug)]
3110pub struct WorkflowHandle {
3111    client: Client,
3112    pub workflow_id: String,
3113    pub run_id: Option<String>,
3114    pub workflow_type: String,
3115}
3116
3117impl WorkflowHandle {
3118    /// Describe whichever run is current for this stable workflow instance.
3119    pub async fn describe(&self) -> Result<WorkflowDescription> {
3120        self.client.describe_workflow(&self.workflow_id).await
3121    }
3122
3123    /// Describe the run identity originally selected by this handle.
3124    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3125        let run_id = self.run_id.as_deref().ok_or_else(|| {
3126            Error::Codec("run_id is required for selected-run description".to_string())
3127        })?;
3128        self.client
3129            .describe_workflow_run(&self.workflow_id, run_id)
3130            .await
3131    }
3132
3133    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3134        self.client
3135            .signal_workflow(&self.workflow_id, signal_name, input)
3136            .await
3137    }
3138
3139    /// Signal only if this handle's selected run is still current.
3140    pub async fn signal_selected_run<T: Serialize>(
3141        &self,
3142        signal_name: &str,
3143        input: T,
3144    ) -> Result<Value> {
3145        let run_id = self.run_id.as_deref().ok_or_else(|| {
3146            Error::Codec("run_id is required for selected-run signaling".to_string())
3147        })?;
3148        self.client
3149            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3150            .await
3151    }
3152
3153    /// Request cooperative cancellation of whichever run is current.
3154    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3155        self.client
3156            .cancel_workflow(&self.workflow_id, options)
3157            .await
3158    }
3159
3160    /// Request cancellation only if this handle's selected run is still current.
3161    pub async fn cancel_selected_run(
3162        &self,
3163        options: WorkflowCommandOptions,
3164    ) -> Result<WorkflowCommandResult> {
3165        let run_id = self.run_id.as_deref().ok_or_else(|| {
3166            Error::Codec("run_id is required for selected-run cancellation".to_string())
3167        })?;
3168        self.client
3169            .cancel_workflow_run(&self.workflow_id, run_id, options)
3170            .await
3171    }
3172
3173    /// Forcefully terminate whichever run is current.
3174    pub async fn terminate(
3175        &self,
3176        options: WorkflowCommandOptions,
3177    ) -> Result<WorkflowCommandResult> {
3178        self.client
3179            .terminate_workflow(&self.workflow_id, options)
3180            .await
3181    }
3182
3183    /// Terminate only if this handle's selected run is still current.
3184    pub async fn terminate_selected_run(
3185        &self,
3186        options: WorkflowCommandOptions,
3187    ) -> Result<WorkflowCommandResult> {
3188        let run_id = self.run_id.as_deref().ok_or_else(|| {
3189            Error::Codec("run_id is required for selected-run termination".to_string())
3190        })?;
3191        self.client
3192            .terminate_workflow_run(&self.workflow_id, run_id, options)
3193            .await
3194    }
3195
3196    /// Execute a named, read-only query against this workflow.
3197    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3198        self.client
3199            .query_workflow(&self.workflow_id, query_name, input)
3200            .await
3201    }
3202
3203    pub async fn query_avro_value<T: Serialize>(
3204        &self,
3205        query_name: &str,
3206        input: T,
3207    ) -> Result<AvroValue> {
3208        self.client
3209            .query_workflow_avro_value(&self.workflow_id, query_name, input)
3210            .await
3211    }
3212
3213    pub async fn update<T: Serialize>(
3214        &self,
3215        update_name: &str,
3216        input: T,
3217        request_id: Option<&str>,
3218    ) -> Result<Value> {
3219        self.client
3220            .update_workflow(&self.workflow_id, update_name, input, request_id)
3221            .await
3222    }
3223
3224    pub async fn update_avro_value<T: Serialize>(
3225        &self,
3226        update_name: &str,
3227        input: T,
3228        request_id: Option<&str>,
3229    ) -> Result<AvroValue> {
3230        self.client
3231            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3232            .await
3233    }
3234
3235    /// Query only if this handle's selected run is still current.
3236    pub async fn query_selected_run<T: Serialize>(
3237        &self,
3238        query_name: &str,
3239        input: T,
3240    ) -> Result<Value> {
3241        let run_id = self
3242            .run_id
3243            .as_deref()
3244            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3245        self.client
3246            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3247            .await
3248    }
3249
3250    /// Await the final terminal outcome of the current continue-as-new chain.
3251    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3252        self.result_target(options, None).await
3253    }
3254
3255    /// Await the final result without projecting Avro bytes through JSON.
3256    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3257        self.result_avro_value_target(options, None).await
3258    }
3259
3260    /// Await only the run identity originally selected by this handle.
3261    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3262        let run_id = self.run_id.as_deref().ok_or_else(|| {
3263            Error::Codec("run_id is required for selected-run result".to_string())
3264        })?;
3265        self.result_target(options, Some(run_id)).await
3266    }
3267
3268    /// Await the selected run's result on the lossless Avro Value surface.
3269    pub async fn result_selected_run_avro_value(
3270        &self,
3271        options: WorkflowResultOptions,
3272    ) -> Result<AvroValue> {
3273        let run_id = self.run_id.as_deref().ok_or_else(|| {
3274            Error::Codec("run_id is required for selected-run result".to_string())
3275        })?;
3276        self.result_avro_value_target(options, Some(run_id)).await
3277    }
3278
3279    async fn result_avro_value_target(
3280        &self,
3281        options: WorkflowResultOptions,
3282        selected_run_id: Option<&str>,
3283    ) -> Result<AvroValue> {
3284        let started = Instant::now();
3285
3286        loop {
3287            let description = match selected_run_id {
3288                Some(run_id) => {
3289                    self.client
3290                        .describe_workflow_run(&self.workflow_id, run_id)
3291                        .await?
3292                }
3293                None => self.describe().await?,
3294            };
3295            if description.is_completed() {
3296                return description.output_avro_value.ok_or_else(|| {
3297                    Error::Codec(
3298                        "missing_payload_envelope: typed workflow result requires output_envelope"
3299                            .to_string(),
3300                    )
3301                });
3302            }
3303            if description.is_terminal() {
3304                let outcome =
3305                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3306                return Err(match outcome.kind {
3307                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3308                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3309                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3310                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3311                });
3312            }
3313            if started.elapsed() >= options.timeout {
3314                return Err(Error::Timeout);
3315            }
3316            tokio::time::sleep(options.poll_interval).await;
3317        }
3318    }
3319
3320    async fn result_target(
3321        &self,
3322        options: WorkflowResultOptions,
3323        selected_run_id: Option<&str>,
3324    ) -> Result<Value> {
3325        let started = Instant::now();
3326
3327        loop {
3328            let description = match selected_run_id {
3329                Some(run_id) => {
3330                    self.client
3331                        .describe_workflow_run(&self.workflow_id, run_id)
3332                        .await?
3333                }
3334                None => self.describe().await?,
3335            };
3336            if description.is_completed() {
3337                return Ok(description.output.unwrap_or(Value::Null));
3338            }
3339
3340            if description.is_terminal() {
3341                let outcome =
3342                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3343                return Err(match outcome.kind {
3344                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3345                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3346                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3347                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3348                });
3349            }
3350
3351            if started.elapsed() >= options.timeout {
3352                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3353                    kind: WorkflowTerminalKind::TimedOut,
3354                    workflow_id: description
3355                        .workflow_id
3356                        .clone()
3357                        .unwrap_or_else(|| self.workflow_id.clone()),
3358                    run_id: description
3359                        .run_id
3360                        .clone()
3361                        .or_else(|| selected_run_id.map(str::to_string)),
3362                    reason: "result_wait_timeout".to_string(),
3363                    failure_category: Some("client_timeout".to_string()),
3364                    failure_id: None,
3365                    exception_type: None,
3366                    exception_class: None,
3367                    non_retryable: None,
3368                    message: Some(format!(
3369                        "workflow result was not terminal within {:?}",
3370                        options.timeout
3371                    )),
3372                    exception: None,
3373                    raw: description.raw_value(),
3374                }));
3375            }
3376
3377            tokio::time::sleep(options.poll_interval).await;
3378        }
3379    }
3380}
3381
3382#[derive(Clone, Copy, Debug)]
3383pub struct WorkflowResultOptions {
3384    pub poll_interval: Duration,
3385    pub timeout: Duration,
3386}
3387
3388impl Default for WorkflowResultOptions {
3389    fn default() -> Self {
3390        Self {
3391            poll_interval: Duration::from_millis(500),
3392            timeout: Duration::from_secs(30),
3393        }
3394    }
3395}
3396
3397#[derive(Clone, Debug, Deserialize)]
3398pub struct WorkflowDescription {
3399    pub workflow_id: Option<String>,
3400    pub run_id: Option<String>,
3401    pub workflow_type: Option<String>,
3402    pub status: Option<String>,
3403    #[serde(default)]
3404    pub closed_reason: Option<String>,
3405    #[serde(default)]
3406    pub error: Option<String>,
3407    #[serde(default)]
3408    pub failure: Option<Value>,
3409    #[serde(default)]
3410    pub exception: Option<Value>,
3411    #[serde(default)]
3412    pub failures: Vec<Value>,
3413    #[serde(default)]
3414    pub output: Option<Value>,
3415    #[serde(default)]
3416    pub output_envelope: Option<Value>,
3417    #[serde(skip)]
3418    pub output_avro_value: Option<AvroValue>,
3419    #[serde(flatten)]
3420    pub raw: HashMap<String, Value>,
3421}
3422
3423impl WorkflowDescription {
3424    pub fn is_completed(&self) -> bool {
3425        matches!(self.status.as_deref(), Some("completed" | "Completed"))
3426    }
3427
3428    pub fn is_terminal(&self) -> bool {
3429        matches!(
3430            self.status.as_deref(),
3431            Some(
3432                "completed"
3433                    | "Completed"
3434                    | "failed"
3435                    | "Failed"
3436                    | "cancelled"
3437                    | "Cancelled"
3438                    | "terminated"
3439                    | "Terminated"
3440                    | "timed_out"
3441                    | "TimedOut",
3442            )
3443        )
3444    }
3445
3446    fn decode_payloads(&mut self) -> Result<()> {
3447        if let Some(envelope) = &self.output_envelope {
3448            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3449            self.output = Some(value.clone().into_json()?);
3450            self.output_avro_value = Some(value);
3451        }
3452
3453        Ok(())
3454    }
3455
3456    fn raw_value(&self) -> Value {
3457        let mut data = self.raw.clone();
3458        data.insert(
3459            "workflow_id".to_string(),
3460            self.workflow_id
3461                .clone()
3462                .map(Value::String)
3463                .unwrap_or(Value::Null),
3464        );
3465        data.insert(
3466            "run_id".to_string(),
3467            self.run_id
3468                .clone()
3469                .map(Value::String)
3470                .unwrap_or(Value::Null),
3471        );
3472        data.insert(
3473            "workflow_type".to_string(),
3474            self.workflow_type
3475                .clone()
3476                .map(Value::String)
3477                .unwrap_or(Value::Null),
3478        );
3479        data.insert(
3480            "status".to_string(),
3481            self.status
3482                .clone()
3483                .map(Value::String)
3484                .unwrap_or(Value::Null),
3485        );
3486        data.insert(
3487            "closed_reason".to_string(),
3488            self.closed_reason
3489                .clone()
3490                .map(Value::String)
3491                .unwrap_or(Value::Null),
3492        );
3493        if let Some(failure) = &self.failure {
3494            data.insert("failure".to_string(), failure.clone());
3495        }
3496        if let Some(exception) = &self.exception {
3497            data.insert("exception".to_string(), exception.clone());
3498        }
3499        Value::Object(data.into_iter().collect())
3500    }
3501}
3502
3503fn workflow_terminal_outcome(
3504    description: &WorkflowDescription,
3505    workflow_id: &str,
3506    run_id: Option<&str>,
3507) -> WorkflowTerminalOutcome {
3508    let terminal_kind = description
3509        .closed_reason
3510        .as_deref()
3511        .or(description.status.as_deref())
3512        .unwrap_or("failed")
3513        .to_ascii_lowercase();
3514    let kind = match terminal_kind.as_str() {
3515        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3516        "terminated" => WorkflowTerminalKind::Terminated,
3517        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3518        _ => WorkflowTerminalKind::Failed,
3519    };
3520    let default_reason = match kind {
3521        WorkflowTerminalKind::Failed => "workflow_failed",
3522        WorkflowTerminalKind::Cancelled => "cancelled",
3523        WorkflowTerminalKind::Terminated => "terminated",
3524        WorkflowTerminalKind::TimedOut => "timed_out",
3525    };
3526    let failure = description
3527        .failure
3528        .as_ref()
3529        .filter(|value| value.is_object());
3530    let nested_failure = failure
3531        .and_then(|value| value.get("failures"))
3532        .and_then(Value::as_array)
3533        .and_then(|failures| failures.last())
3534        .or_else(|| description.failures.last());
3535    let exception = description
3536        .exception
3537        .clone()
3538        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3539        .or_else(|| {
3540            nested_failure
3541                .and_then(|value| value.get("exception_payload"))
3542                .cloned()
3543        });
3544    let string_field = |name: &str| {
3545        failure
3546            .and_then(|value| value.get(name))
3547            .and_then(Value::as_str)
3548            .or_else(|| {
3549                nested_failure
3550                    .and_then(|value| value.get(name))
3551                    .and_then(Value::as_str)
3552            })
3553            .map(str::to_string)
3554    };
3555    let exception_field = |name: &str| {
3556        exception
3557            .as_ref()
3558            .and_then(|value| value.get(name))
3559            .and_then(Value::as_str)
3560            .map(str::to_string)
3561    };
3562    let message = description
3563        .error
3564        .clone()
3565        .or_else(|| string_field("message"))
3566        .or_else(|| exception_field("message"));
3567    let reason = description
3568        .raw
3569        .get("reason")
3570        .and_then(Value::as_str)
3571        .map(str::to_string)
3572        .or_else(|| {
3573            failure
3574                .and_then(|value| value.get("reason"))
3575                .and_then(Value::as_str)
3576                .map(str::to_string)
3577        })
3578        .or_else(|| description.closed_reason.clone())
3579        .unwrap_or_else(|| default_reason.to_string());
3580    let failure_id = string_field("failure_id").or_else(|| {
3581        nested_failure
3582            .and_then(|value| value.get("id"))
3583            .and_then(Value::as_str)
3584            .map(str::to_string)
3585    });
3586
3587    WorkflowTerminalOutcome {
3588        kind,
3589        workflow_id: description
3590            .workflow_id
3591            .clone()
3592            .unwrap_or_else(|| workflow_id.to_string()),
3593        run_id: description
3594            .run_id
3595            .clone()
3596            .or_else(|| run_id.map(str::to_string)),
3597        reason,
3598        failure_category: string_field("failure_category")
3599            .or_else(|| Some(default_reason.to_string())),
3600        failure_id,
3601        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3602        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3603        non_retryable: failure
3604            .and_then(|value| value.get("non_retryable"))
3605            .and_then(Value::as_bool)
3606            .or_else(|| {
3607                nested_failure
3608                    .and_then(|value| value.get("non_retryable"))
3609                    .and_then(Value::as_bool)
3610            }),
3611        message,
3612        exception,
3613        raw: description.raw_value(),
3614    }
3615}
3616
3617#[derive(Clone, Debug, Deserialize)]
3618pub struct RegisterWorkerResponse {
3619    pub worker_id: String,
3620    pub registered: bool,
3621    #[serde(default)]
3622    pub heartbeat_interval_seconds: Option<u64>,
3623    #[serde(default)]
3624    pub protocol_version: Option<String>,
3625    #[serde(default)]
3626    pub server_capabilities: Option<Value>,
3627}
3628
3629/// Result of gracefully removing a worker-plane registration.
3630#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3631pub struct WorkerDeregistrationEnvelope {
3632    pub worker_id: String,
3633    pub outcome: String,
3634    pub recovered_workflow_task_count: u64,
3635}
3636
3637#[derive(Clone, Debug, Deserialize)]
3638pub struct PollWorkflowTaskResponse {
3639    #[serde(default)]
3640    pub task: Option<WorkflowTask>,
3641    #[serde(default)]
3642    pub poll_status: Option<String>,
3643    #[serde(default)]
3644    pub reason: Option<String>,
3645    #[serde(default)]
3646    pub protocol_version: Option<String>,
3647    #[serde(default)]
3648    pub server_capabilities: Option<Value>,
3649}
3650
3651impl PollWorkflowTaskResponse {
3652    /// Classify this response without parsing server display text.
3653    pub fn outcome(&self) -> WorkerPollOutcome {
3654        worker_poll_outcome(
3655            self.task.is_some(),
3656            self.poll_status.as_deref(),
3657            self.reason.as_deref(),
3658        )
3659    }
3660}
3661
3662#[derive(Clone, Debug, Deserialize)]
3663pub struct PollActivityTaskResponse {
3664    #[serde(default)]
3665    pub task: Option<ActivityTask>,
3666    #[serde(default)]
3667    pub poll_status: Option<String>,
3668    #[serde(default)]
3669    pub reason: Option<String>,
3670}
3671
3672impl PollActivityTaskResponse {
3673    /// Classify this response without parsing server display text.
3674    pub fn outcome(&self) -> WorkerPollOutcome {
3675        worker_poll_outcome(
3676            self.task.is_some(),
3677            self.poll_status.as_deref(),
3678            self.reason.as_deref(),
3679        )
3680    }
3681}
3682
3683#[derive(Clone, Debug, Deserialize)]
3684pub struct PollQueryTaskResponse {
3685    #[serde(default)]
3686    pub task: Option<QueryTask>,
3687    #[serde(default)]
3688    pub poll_status: Option<String>,
3689    #[serde(default)]
3690    pub reason: Option<String>,
3691}
3692
3693impl PollQueryTaskResponse {
3694    /// Classify this response without parsing server display text.
3695    pub fn outcome(&self) -> WorkerPollOutcome {
3696        worker_poll_outcome(
3697            self.task.is_some(),
3698            self.poll_status.as_deref(),
3699            self.reason.as_deref(),
3700        )
3701    }
3702}
3703
3704/// Stable classification for worker poll responses.
3705#[derive(Clone, Debug, PartialEq, Eq)]
3706pub enum WorkerPollOutcome {
3707    /// A task was leased and is available on the response.
3708    Task,
3709    /// No task was leased, but the worker should continue polling.
3710    Idle {
3711        poll_status: Option<String>,
3712        reason: Option<String>,
3713    },
3714    /// The server asked this worker to stop claiming new work.
3715    Stop {
3716        poll_status: Option<String>,
3717        reason: Option<String>,
3718    },
3719}
3720
3721impl WorkerPollOutcome {
3722    pub fn should_stop(&self) -> bool {
3723        matches!(self, Self::Stop { .. })
3724    }
3725}
3726
3727fn worker_poll_outcome(
3728    has_task: bool,
3729    poll_status: Option<&str>,
3730    reason: Option<&str>,
3731) -> WorkerPollOutcome {
3732    if worker_poll_is_stop(poll_status, reason) {
3733        return WorkerPollOutcome::Stop {
3734            poll_status: poll_status.map(str::to_string),
3735            reason: reason.map(str::to_string),
3736        };
3737    }
3738
3739    if has_task {
3740        WorkerPollOutcome::Task
3741    } else {
3742        WorkerPollOutcome::Idle {
3743            poll_status: poll_status.map(str::to_string),
3744            reason: reason.map(str::to_string),
3745        }
3746    }
3747}
3748
3749/// An ephemeral server-routed query task.
3750#[derive(Clone, Debug, Deserialize)]
3751pub struct QueryTask {
3752    pub query_task_id: String,
3753    #[serde(default = "default_workflow_task_attempt")]
3754    pub query_task_attempt: u64,
3755    #[serde(default)]
3756    pub lease_owner: Option<String>,
3757    #[serde(default)]
3758    pub workflow_id: Option<String>,
3759    #[serde(default)]
3760    pub run_id: Option<String>,
3761    pub workflow_type: String,
3762    pub query_name: String,
3763    #[serde(default = "default_payload_codec")]
3764    pub payload_codec: String,
3765    #[serde(default)]
3766    pub workflow_arguments: Option<Value>,
3767    #[serde(default)]
3768    pub query_arguments: Option<Value>,
3769    #[serde(default)]
3770    pub history_events: Vec<HistoryEvent>,
3771    #[serde(default)]
3772    pub history_export: Option<Value>,
3773    #[serde(default)]
3774    pub run_status: Option<String>,
3775}
3776
3777#[derive(Clone, Debug, Deserialize)]
3778pub struct WorkflowTask {
3779    pub task_id: String,
3780    #[serde(default)]
3781    pub workflow_id: Option<String>,
3782    #[serde(default)]
3783    pub run_id: Option<String>,
3784    pub workflow_type: String,
3785    #[serde(default = "default_payload_codec")]
3786    pub payload_codec: String,
3787    #[serde(default)]
3788    pub arguments: Option<Value>,
3789    #[serde(default)]
3790    pub history_events: Vec<HistoryEvent>,
3791    #[serde(default)]
3792    pub total_history_events: Option<u64>,
3793    #[serde(default)]
3794    pub history_size_bytes: Option<u64>,
3795    #[serde(default)]
3796    pub continue_as_new_recommended: Option<bool>,
3797    #[serde(default)]
3798    pub history_budget_pressure: Option<String>,
3799    #[serde(default)]
3800    pub next_history_page_token: Option<String>,
3801    #[serde(default = "default_workflow_task_attempt")]
3802    pub workflow_task_attempt: u64,
3803    #[serde(default)]
3804    pub workflow_signal_id: Option<String>,
3805    #[serde(default)]
3806    pub signal_name: Option<String>,
3807    #[serde(default)]
3808    pub signal_arguments: Option<Value>,
3809    #[serde(default)]
3810    pub workflow_update_id: Option<String>,
3811    #[serde(default)]
3812    pub update_name: Option<String>,
3813    #[serde(default)]
3814    pub lease_owner: Option<String>,
3815}
3816
3817impl WorkflowTask {
3818    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3819        self.history_events.extend(page.history_events);
3820
3821        if page.total_history_events.is_some() {
3822            self.total_history_events = page.total_history_events;
3823        }
3824
3825        self.next_history_page_token = page
3826            .next_history_page_token
3827            .filter(|token| !token.is_empty());
3828    }
3829}
3830
3831#[derive(Clone, Debug, Deserialize)]
3832struct WorkflowTaskHistoryPage {
3833    #[serde(default)]
3834    history_events: Vec<HistoryEvent>,
3835    #[serde(default)]
3836    total_history_events: Option<u64>,
3837    #[serde(default)]
3838    next_history_page_token: Option<String>,
3839}
3840
3841#[derive(Clone, Debug, Deserialize)]
3842pub struct ActivityTask {
3843    pub task_id: String,
3844    #[serde(default)]
3845    pub activity_attempt_id: Option<String>,
3846    #[serde(default)]
3847    pub attempt_id: Option<String>,
3848    pub activity_type: String,
3849    #[serde(default = "default_payload_codec")]
3850    pub payload_codec: String,
3851    #[serde(default)]
3852    pub arguments: Option<Value>,
3853    #[serde(default = "default_attempt_number")]
3854    pub attempt_number: u64,
3855    #[serde(default)]
3856    pub lease_owner: Option<String>,
3857}
3858
3859#[derive(Clone, Debug, Deserialize)]
3860pub struct HistoryEvent {
3861    #[serde(alias = "type")]
3862    pub event_type: String,
3863    #[serde(default)]
3864    pub payload: Value,
3865    #[serde(flatten)]
3866    pub raw: HashMap<String, Value>,
3867}
3868
3869/// One decoded signal in the committed workflow-history snapshot.
3870#[derive(Clone, Debug, PartialEq)]
3871pub struct QuerySignal {
3872    pub id: Option<String>,
3873    pub name: String,
3874    pub arguments: Vec<Value>,
3875    avro_arguments: Vec<AvroValue>,
3876    pub workflow_sequence: Option<u64>,
3877}
3878
3879impl QuerySignal {
3880    /// Lossless fixed Avro Value arguments for this committed signal.
3881    pub fn arguments_avro_value(&self) -> &[AvroValue] {
3882        &self.avro_arguments
3883    }
3884}
3885
3886/// Immutable state supplied to a registered query handler.
3887///
3888/// This context intentionally exposes no activity, signal-wait, or command
3889/// APIs. Query handlers inspect committed history and return a value; query
3890/// completion does not append an event or advance deterministic execution.
3891#[derive(Clone, Debug)]
3892pub struct QueryContext {
3893    pub workflow_id: Option<String>,
3894    pub run_id: Option<String>,
3895    pub workflow_type: String,
3896    pub run_status: Option<String>,
3897    workflow_input: Value,
3898    workflow_input_avro_value: AvroValue,
3899    history_events: Arc<Vec<HistoryEvent>>,
3900    signal_events: Arc<Vec<QuerySignal>>,
3901}
3902
3903impl QueryContext {
3904    /// The normalized argument list used to start the workflow.
3905    pub fn workflow_input(&self) -> &Value {
3906        &self.workflow_input
3907    }
3908
3909    /// The lossless fixed Avro Value argument list used to start the workflow.
3910    pub fn workflow_input_avro_value(&self) -> &AvroValue {
3911        &self.workflow_input_avro_value
3912    }
3913
3914    /// The immutable committed history used for this query snapshot.
3915    pub fn history_events(&self) -> &[HistoryEvent] {
3916        self.history_events.as_slice()
3917    }
3918
3919    /// All decoded signals in committed workflow order.
3920    pub fn signal_events(&self) -> &[QuerySignal] {
3921        self.signal_events.as_slice()
3922    }
3923
3924    /// Decoded argument lists for each committed signal with `signal_name`.
3925    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3926        self.signal_events
3927            .iter()
3928            .filter(|signal| signal.name == signal_name)
3929            .map(|signal| signal.arguments.clone())
3930            .collect()
3931    }
3932
3933    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
3934    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3935        self.signal_events
3936            .iter()
3937            .filter(|signal| signal.name == signal_name)
3938            .map(|signal| signal.avro_arguments.clone())
3939            .collect()
3940    }
3941}
3942
3943#[derive(Clone, Debug, Deserialize)]
3944pub struct ActivityHeartbeatResponse {
3945    #[serde(default)]
3946    pub cancel_requested: bool,
3947    #[serde(default)]
3948    pub heartbeat_recorded: bool,
3949    #[serde(default)]
3950    pub can_continue: Option<bool>,
3951    #[serde(default)]
3952    pub reason: Option<String>,
3953    #[serde(default)]
3954    pub run_closed_reason: Option<String>,
3955    #[serde(default)]
3956    pub run_closed_at: Option<String>,
3957    #[serde(default)]
3958    pub lease_expires_at: Option<String>,
3959    #[serde(default)]
3960    pub last_heartbeat_at: Option<String>,
3961}
3962
3963impl ActivityHeartbeatResponse {
3964    /// Whether the activity should stop instead of attempting completion.
3965    pub fn should_stop(&self) -> bool {
3966        self.cancel_requested || self.can_continue == Some(false)
3967    }
3968}
3969
3970fn default_payload_codec() -> String {
3971    DEFAULT_CODEC.to_string()
3972}
3973
3974fn default_workflow_task_attempt() -> u64 {
3975    1
3976}
3977
3978fn default_attempt_number() -> u64 {
3979    1
3980}
3981
3982type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3983type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
3984type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
3985type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
3986type ReplayedWorkflowHandler =
3987    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
3988type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3989type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
3990type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3991type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3992type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3993type ReplayedQueryHandler = Arc<
3994    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
3995        + Send
3996        + Sync,
3997>;
3998type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
3999
4000struct ReplayedWorkflowInvocation {
4001    future: WorkflowFuture,
4002    snapshot: WorkflowStateSnapshot,
4003}
4004
4005#[derive(Clone)]
4006struct RegisteredWorkflow {
4007    execute: WorkflowHandler,
4008    replay: Option<ReplayedWorkflowHandler>,
4009    state_type: Option<TypeId>,
4010}
4011
4012#[derive(Clone)]
4013enum RegisteredQuery {
4014    Snapshot(QueryHandler),
4015    Replayed {
4016        state_type: TypeId,
4017        handler: ReplayedQueryHandler,
4018    },
4019}
4020
4021#[derive(Clone, Debug)]
4022pub struct WorkerHeartbeatObservation {
4023    pub worker_id: String,
4024    pub task_queue: String,
4025    pub acknowledged_at_unix_millis: u64,
4026    pub acknowledgement: Value,
4027}
4028
4029/// Bounded retry policy for worker poll acquisition and worker heartbeats.
4030///
4031/// Expected empty long polls are normal successful responses. Transport
4032/// failures, HTTP 408/429 responses, and server errors are retried with capped
4033/// exponential backoff. Authentication, protocol, codec, and handler failures
4034/// are never retried by the worker.
4035#[derive(Clone, Copy, Debug)]
4036pub struct WorkerRetryPolicy {
4037    /// Number of retries after the initial request fails.
4038    pub max_retries: usize,
4039    /// Delay before the first retry.
4040    pub initial_backoff: Duration,
4041    /// Maximum delay between retries.
4042    pub max_backoff: Duration,
4043}
4044
4045impl Default for WorkerRetryPolicy {
4046    fn default() -> Self {
4047        Self {
4048            max_retries: 5,
4049            initial_backoff: Duration::from_millis(100),
4050            max_backoff: Duration::from_secs(5),
4051        }
4052    }
4053}
4054
4055#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4056enum ManagedPollOutcome {
4057    Idle,
4058    Handled,
4059    Stop,
4060}
4061
4062#[derive(Clone)]
4063pub struct Worker {
4064    client: Client,
4065    worker_id: String,
4066    task_queue: String,
4067    workflows: HashMap<String, RegisteredWorkflow>,
4068    activities: HashMap<String, ActivityHandler>,
4069    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4070    updates: HashMap<String, HashMap<String, UpdateHandler>>,
4071    max_concurrent_workflow_tasks: usize,
4072    max_concurrent_activity_tasks: usize,
4073    poll_timeout: Duration,
4074    heartbeat_interval: Duration,
4075    retry_policy: WorkerRetryPolicy,
4076    heartbeat_observer: Option<WorkerHeartbeatObserver>,
4077}
4078
4079impl Worker {
4080    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4081        Self {
4082            client,
4083            worker_id: default_worker_id(),
4084            task_queue: task_queue.into(),
4085            workflows: HashMap::new(),
4086            activities: HashMap::new(),
4087            queries: HashMap::new(),
4088            updates: HashMap::new(),
4089            max_concurrent_workflow_tasks: 10,
4090            max_concurrent_activity_tasks: 10,
4091            poll_timeout: Duration::from_secs(30),
4092            heartbeat_interval: Duration::from_secs(60),
4093            retry_policy: WorkerRetryPolicy::default(),
4094            heartbeat_observer: None,
4095        }
4096    }
4097
4098    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4099        self.worker_id = worker_id.into();
4100        self
4101    }
4102
4103    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4104        self.poll_timeout = timeout;
4105        self
4106    }
4107
4108    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4109        self.heartbeat_interval = interval;
4110        self
4111    }
4112
4113    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
4114    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4115        self.retry_policy = policy;
4116        self
4117    }
4118
4119    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4120    where
4121        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4122    {
4123        self.heartbeat_observer = Some(Arc::new(observer));
4124        self
4125    }
4126
4127    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4128        self.max_concurrent_workflow_tasks = count.max(1);
4129        self
4130    }
4131
4132    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4133        self.max_concurrent_activity_tasks = count.max(1);
4134        self
4135    }
4136
4137    /// Register a workflow handler.
4138    ///
4139    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
4140    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
4141    /// while acquiring or decoding a worker task remain worker-operation
4142    /// failures and do not get converted into workflow outcomes.
4143    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4144    where
4145        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4146        Fut: Future<Output = Result<Value>> + Send + 'static,
4147    {
4148        let handler = Arc::new(handler);
4149        self.workflows.insert(
4150            workflow_type.into(),
4151            RegisteredWorkflow {
4152                execute: Arc::new(move |ctx, input| {
4153                    let handler = Arc::clone(&handler);
4154                    Box::pin(async move {
4155                        let result = handler(ctx, input.into_json()?).await?;
4156                        AvroValue::from_serialize(&result)
4157                    })
4158                }),
4159                replay: None,
4160                state_type: None,
4161            },
4162        );
4163    }
4164
4165    /// Register a workflow on the lossless fixed Avro Value surface.
4166    pub fn register_workflow_avro_value<F, Fut>(
4167        &mut self,
4168        workflow_type: impl Into<String>,
4169        handler: F,
4170    ) where
4171        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4172        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4173    {
4174        self.workflows.insert(
4175            workflow_type.into(),
4176            RegisteredWorkflow {
4177                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4178                replay: None,
4179                state_type: None,
4180            },
4181        );
4182    }
4183
4184    /// Register a workflow whose typed instance state can be reconstructed for queries.
4185    ///
4186    /// `state_factory` creates a fresh instance for every normal workflow task and
4187    /// query replay. The workflow handler is the single source of truth for state
4188    /// transitions: it updates [`WorkflowInstance`] after activities and signals
4189    /// resolve. Query replay runs this same handler over committed history and
4190    /// discards any commands it would emit.
4191    pub fn register_replayed_workflow<S, Factory, F, Fut>(
4192        &mut self,
4193        workflow_type: impl Into<String>,
4194        state_factory: Factory,
4195        handler: F,
4196    ) where
4197        S: Clone + Send + Sync + 'static,
4198        Factory: Fn() -> S + Send + Sync + 'static,
4199        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4200        Fut: Future<Output = Result<Value>> + Send + 'static,
4201    {
4202        let state_factory = Arc::new(state_factory);
4203        let handler = Arc::new(handler);
4204
4205        let execute_factory = Arc::clone(&state_factory);
4206        let execute_handler = Arc::clone(&handler);
4207        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4208            let state = WorkflowInstance::new(execute_factory());
4209            let handler = Arc::clone(&execute_handler);
4210            Box::pin(async move {
4211                let result = handler(ctx, input.into_json()?, state).await?;
4212                AvroValue::from_serialize(&result)
4213            }) as WorkflowFuture
4214        });
4215
4216        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4217            let state = WorkflowInstance::new(state_factory());
4218            let snapshot_state = state.clone();
4219            let snapshot: WorkflowStateSnapshot =
4220                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4221            let replay_handler = Arc::clone(&handler);
4222            let future = async move {
4223                let result = replay_handler(ctx, input.into_json()?, state).await?;
4224                AvroValue::from_serialize(&result)
4225            };
4226            ReplayedWorkflowInvocation {
4227                future: Box::pin(future),
4228                snapshot,
4229            }
4230        });
4231
4232        self.workflows.insert(
4233            workflow_type.into(),
4234            RegisteredWorkflow {
4235                execute,
4236                replay: Some(replay),
4237                state_type: Some(TypeId::of::<S>()),
4238            },
4239        );
4240    }
4241
4242    /// Register a replayable workflow on the lossless fixed Avro Value surface.
4243    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4244        &mut self,
4245        workflow_type: impl Into<String>,
4246        state_factory: Factory,
4247        handler: F,
4248    ) where
4249        S: Clone + Send + Sync + 'static,
4250        Factory: Fn() -> S + Send + Sync + 'static,
4251        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4252        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4253    {
4254        let state_factory = Arc::new(state_factory);
4255        let handler = Arc::new(handler);
4256
4257        let execute_factory = Arc::clone(&state_factory);
4258        let execute_handler = Arc::clone(&handler);
4259        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4260            let state = WorkflowInstance::new(execute_factory());
4261            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4262        });
4263
4264        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4265            let state = WorkflowInstance::new(state_factory());
4266            let snapshot_state = state.clone();
4267            let snapshot: WorkflowStateSnapshot =
4268                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4269            ReplayedWorkflowInvocation {
4270                future: Box::pin(handler(ctx, input, state)),
4271                snapshot,
4272            }
4273        });
4274
4275        self.workflows.insert(
4276            workflow_type.into(),
4277            RegisteredWorkflow {
4278                execute,
4279                replay: Some(replay),
4280                state_type: Some(TypeId::of::<S>()),
4281            },
4282        );
4283    }
4284
4285    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4286    where
4287        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4288        Fut: Future<Output = Result<Value>> + Send + 'static,
4289    {
4290        let handler = Arc::new(handler);
4291        self.activities.insert(
4292            activity_type.into(),
4293            Arc::new(move |ctx, args| {
4294                let handler = Arc::clone(&handler);
4295                Box::pin(async move {
4296                    let result = handler(ctx, args.into_json()?).await?;
4297                    AvroValue::from_serialize(&result)
4298                })
4299            }),
4300        );
4301    }
4302
4303    /// Register an activity on the lossless fixed Avro Value surface.
4304    pub fn register_activity_avro_value<F, Fut>(
4305        &mut self,
4306        activity_type: impl Into<String>,
4307        handler: F,
4308    ) where
4309        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4310        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4311    {
4312        self.activities.insert(
4313            activity_type.into(),
4314            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4315        );
4316    }
4317
4318    /// Register a named, read-only query handler for a workflow type.
4319    ///
4320    /// The workflow type must also be registered with [`Worker::register_workflow`]
4321    /// before the worker runs. The handler receives only an immutable committed
4322    /// state snapshot and normalized query arguments.
4323    pub fn register_query<F, Fut>(
4324        &mut self,
4325        workflow_type: impl Into<String>,
4326        query_name: impl Into<String>,
4327        handler: F,
4328    ) where
4329        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4330        Fut: Future<Output = Result<Value>> + Send + 'static,
4331    {
4332        let handler = Arc::new(handler);
4333        self.queries
4334            .entry(workflow_type.into())
4335            .or_default()
4336            .insert(
4337                query_name.into(),
4338                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4339                    let handler = Arc::clone(&handler);
4340                    Box::pin(async move {
4341                        let result = handler(ctx, args.into_json()?).await?;
4342                        AvroValue::from_serialize(&result)
4343                    })
4344                })),
4345            );
4346    }
4347
4348    /// Register a query handler on the lossless fixed Avro Value surface.
4349    pub fn register_query_avro_value<F, Fut>(
4350        &mut self,
4351        workflow_type: impl Into<String>,
4352        query_name: impl Into<String>,
4353        handler: F,
4354    ) where
4355        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4356        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4357    {
4358        self.queries
4359            .entry(workflow_type.into())
4360            .or_default()
4361            .insert(
4362                query_name.into(),
4363                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4364            );
4365    }
4366
4367    /// Register a named query against deterministically replayed instance state.
4368    ///
4369    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
4370    /// same state type `S`. The handler receives an immutable, detached state
4371    /// clone, so successful and failed queries cannot affect workflow execution
4372    /// or the state reconstructed by a later query.
4373    pub fn register_replayed_query<S, F, Fut>(
4374        &mut self,
4375        workflow_type: impl Into<String>,
4376        query_name: impl Into<String>,
4377        handler: F,
4378    ) where
4379        S: Clone + Send + Sync + 'static,
4380        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4381        Fut: Future<Output = Result<Value>> + Send + 'static,
4382    {
4383        let handler = Arc::new(handler);
4384        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4385            let state = state.downcast::<S>().map_err(|_| {
4386                "registered query state type does not match the replayed workflow state".to_string()
4387            })?;
4388            let handler = Arc::clone(&handler);
4389            Ok(Box::pin(async move {
4390                let result = handler(ctx, state, args.into_json()?).await?;
4391                AvroValue::from_serialize(&result)
4392            }))
4393        });
4394
4395        self.queries
4396            .entry(workflow_type.into())
4397            .or_default()
4398            .insert(
4399                query_name.into(),
4400                RegisteredQuery::Replayed {
4401                    state_type: TypeId::of::<S>(),
4402                    handler: erased_handler,
4403                },
4404            );
4405    }
4406
4407    /// Register a replayed-state query on the lossless fixed Avro Value surface.
4408    pub fn register_replayed_query_avro_value<S, F, Fut>(
4409        &mut self,
4410        workflow_type: impl Into<String>,
4411        query_name: impl Into<String>,
4412        handler: F,
4413    ) where
4414        S: Clone + Send + Sync + 'static,
4415        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4416        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4417    {
4418        let handler = Arc::new(handler);
4419        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4420            let state = state.downcast::<S>().map_err(|_| {
4421                "registered query state type does not match the replayed workflow state".to_string()
4422            })?;
4423            Ok(Box::pin(handler(ctx, state, args)))
4424        });
4425
4426        self.queries
4427            .entry(workflow_type.into())
4428            .or_default()
4429            .insert(
4430                query_name.into(),
4431                RegisteredQuery::Replayed {
4432                    state_type: TypeId::of::<S>(),
4433                    handler: erased_handler,
4434                },
4435            );
4436    }
4437
4438    /// Register a synchronous workflow update handler.
4439    pub fn register_update<F, Fut>(
4440        &mut self,
4441        workflow_type: impl Into<String>,
4442        update_name: impl Into<String>,
4443        handler: F,
4444    ) where
4445        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4446        Fut: Future<Output = Result<Value>> + Send + 'static,
4447    {
4448        let handler = Arc::new(handler);
4449        self.updates
4450            .entry(workflow_type.into())
4451            .or_default()
4452            .insert(
4453                update_name.into(),
4454                Arc::new(move |ctx, args| {
4455                    let handler = Arc::clone(&handler);
4456                    Box::pin(async move {
4457                        let result = handler(ctx, args.into_json()?).await?;
4458                        AvroValue::from_serialize(&result)
4459                    })
4460                }),
4461            );
4462    }
4463
4464    /// Register an update handler on the lossless fixed Avro Value surface.
4465    pub fn register_update_avro_value<F, Fut>(
4466        &mut self,
4467        workflow_type: impl Into<String>,
4468        update_name: impl Into<String>,
4469        handler: F,
4470    ) where
4471        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4472        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4473    {
4474        self.updates
4475            .entry(workflow_type.into())
4476            .or_default()
4477            .insert(
4478                update_name.into(),
4479                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4480            );
4481    }
4482
4483    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4484        let mut command_contracts = serde_json::Map::new();
4485        for workflow_type in self.workflows.keys() {
4486            let mut queries = self
4487                .queries
4488                .get(workflow_type)
4489                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4490                .unwrap_or_default();
4491            queries.sort();
4492            let mut updates = self
4493                .updates
4494                .get(workflow_type)
4495                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4496                .unwrap_or_default();
4497            updates.sort();
4498            if !queries.is_empty() || !updates.is_empty() {
4499                command_contracts.insert(
4500                    workflow_type.clone(),
4501                    json!({
4502                        "queries": queries,
4503                        "updates": updates,
4504                        "update_validators": [],
4505                    }),
4506                );
4507            }
4508        }
4509
4510        self.client
4511            .register_worker_with_command_contracts(
4512                &self.worker_id,
4513                &self.task_queue,
4514                self.workflows.keys().cloned().collect(),
4515                self.activities.keys().cloned().collect(),
4516                self.max_concurrent_workflow_tasks,
4517                self.max_concurrent_activity_tasks,
4518                [
4519                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4520                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4521                ]
4522                .into_iter()
4523                .flatten()
4524                .collect(),
4525                Value::Object(command_contracts),
4526            )
4527            .await
4528    }
4529
4530    /// Run until shutdown or a terminal worker error occurs.
4531    ///
4532    /// Empty long-poll expirations do not stop the worker. Retryable poll and
4533    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
4534    /// authentication, protocol, and other non-retryable failures are returned.
4535    pub async fn run(&self) -> Result<()> {
4536        self.run_until(std::future::pending::<()>()).await
4537    }
4538
4539    /// Run until `shutdown` resolves or a terminal worker error occurs.
4540    ///
4541    /// This has the same liveness and terminal-error contract as [`Worker::run`].
4542    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4543    where
4544        F: Future<Output = ()>,
4545    {
4546        let registration = self.register().await?;
4547        if !registration.registered {
4548            return Err(Error::WorkerLoop(format!(
4549                "worker registration for {:?} was not accepted",
4550                self.worker_id
4551            )));
4552        }
4553        let registered_worker_id = registration.worker_id.clone();
4554        let primary = self.run_registered_until(shutdown, registration).await;
4555        let deregistration = self
4556            .client
4557            .deregister_worker_registration(&registered_worker_id)
4558            .await;
4559
4560        match (primary, deregistration) {
4561            (Ok(()), Ok(_)) => Ok(()),
4562            (Ok(()), Err(deregistration)) => Err(deregistration),
4563            (Err(primary), Ok(_)) => Err(primary),
4564            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4565                primary: Box::new(primary),
4566                deregistration: Box::new(deregistration),
4567            }),
4568        }
4569    }
4570
4571    async fn run_registered_until<F>(
4572        &self,
4573        shutdown: F,
4574        registration: RegisterWorkerResponse,
4575    ) -> Result<()>
4576    where
4577        F: Future<Output = ()>,
4578    {
4579        let heartbeat_interval = Duration::from_secs(
4580            registration
4581                .heartbeat_interval_seconds
4582                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4583        );
4584        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
4585        // from the completion of the preceding attempt, including its bounded
4586        // retries. A fixed-epoch interval can leave an already-due tick queued
4587        // while an acknowledgement is slow, producing a catch-up heartbeat as
4588        // soon as that request completes.
4589        let heartbeat = tokio::time::sleep(Duration::ZERO);
4590        tokio::pin!(heartbeat);
4591        tokio::pin!(shutdown);
4592        let stop = Arc::new(AtomicBool::new(false));
4593        // Poll responses may already have leased server-side work by the time
4594        // they become ready, so each poller owns its responses through
4595        // completion or failure instead of racing raw polls in this select.
4596        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4597            let worker = self.clone();
4598            let stop = Arc::clone(&stop);
4599            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4600        });
4601        let mut activity_poller = (!self.activities.is_empty()).then(|| {
4602            let worker = self.clone();
4603            let stop = Arc::clone(&stop);
4604            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4605        });
4606        let mut query_poller = (!self.queries.is_empty()).then(|| {
4607            let worker = self.clone();
4608            let stop = Arc::clone(&stop);
4609            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4610        });
4611
4612        loop {
4613            tokio::select! {
4614                _ = &mut shutdown => {
4615                    stop.store(true, Ordering::SeqCst);
4616                    break;
4617                }
4618                _ = &mut heartbeat => {
4619                    let result = self.retry_worker_operation(|| {
4620                        self.client.heartbeat_worker(
4621                            &self.worker_id,
4622                            self.max_concurrent_workflow_tasks,
4623                            self.max_concurrent_activity_tasks,
4624                        )
4625                    }).await;
4626                    heartbeat
4627                        .as_mut()
4628                        .reset(tokio::time::Instant::now() + heartbeat_interval);
4629                    match result {
4630                        Ok(acknowledgement) => {
4631                            if let Some(observer) = &self.heartbeat_observer {
4632                                observer(&WorkerHeartbeatObservation {
4633                                    worker_id: self.worker_id.clone(),
4634                                    task_queue: self.task_queue.clone(),
4635                                    acknowledged_at_unix_millis: SystemTime::now()
4636                                        .duration_since(UNIX_EPOCH)
4637                                        .unwrap_or_default()
4638                                        .as_millis()
4639                                        .min(u64::MAX as u128)
4640                                        as u64,
4641                                    acknowledgement,
4642                                });
4643                            }
4644                        }
4645                        Err(error) => {
4646                            stop.store(true, Ordering::SeqCst);
4647                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4648                            return Err(error);
4649                        }
4650                    }
4651                }
4652                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4653                    workflow_poller = None;
4654                    let stopped_by_server = stop.load(Ordering::SeqCst);
4655                    stop.store(true, Ordering::SeqCst);
4656                    let poller_result = optional_poller_result("workflow", result);
4657                    let join_result =
4658                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4659                    poller_result?;
4660                    join_result?;
4661                    if stopped_by_server {
4662                        return Ok(());
4663                    }
4664                    return Err(Error::WorkerLoop(
4665                        "workflow poller stopped unexpectedly".to_string(),
4666                    ));
4667                }
4668                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4669                    activity_poller = None;
4670                    let stopped_by_server = stop.load(Ordering::SeqCst);
4671                    stop.store(true, Ordering::SeqCst);
4672                    let poller_result = optional_poller_result("activity", result);
4673                    let join_result =
4674                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4675                    poller_result?;
4676                    join_result?;
4677                    if stopped_by_server {
4678                        return Ok(());
4679                    }
4680                    return Err(Error::WorkerLoop(
4681                        "activity poller stopped unexpectedly".to_string(),
4682                    ));
4683                }
4684                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4685                    query_poller = None;
4686                    let stopped_by_server = stop.load(Ordering::SeqCst);
4687                    stop.store(true, Ordering::SeqCst);
4688                    let poller_result = optional_poller_result("query", result);
4689                    let join_result =
4690                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4691                    poller_result?;
4692                    join_result?;
4693                    if stopped_by_server {
4694                        return Ok(());
4695                    }
4696                    return Err(Error::WorkerLoop(
4697                        "query poller stopped unexpectedly".to_string(),
4698                    ));
4699                }
4700            }
4701        }
4702
4703        join_pollers(
4704            workflow_poller.take(),
4705            activity_poller.take(),
4706            query_poller.take(),
4707        )
4708        .await
4709    }
4710
4711    /// Poll and settle at most one task from each enabled task family.
4712    ///
4713    /// A workflow may reach its server-enforced run deadline while this worker
4714    /// holds a task. When the completion endpoint authoritatively rejects that
4715    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
4716    /// terminal `run_status=failed`, the workflow tick is considered settled:
4717    /// the late command was not recorded and cannot replace the terminal run.
4718    /// Every other completion rejection remains an error. This worker-level
4719    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
4720    /// only bounds how long a client waits for a result.
4721    ///
4722    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
4723    /// the original [`Error::Http`] status and response body.
4724    pub async fn run_once(&self) -> Result<usize> {
4725        let mut handled = 0;
4726        match self.poll_workflow_once().await? {
4727            ManagedPollOutcome::Handled => handled += 1,
4728            ManagedPollOutcome::Stop => return Ok(handled),
4729            ManagedPollOutcome::Idle => {}
4730        }
4731        match self.poll_activity_once().await? {
4732            ManagedPollOutcome::Handled => handled += 1,
4733            ManagedPollOutcome::Stop => return Ok(handled),
4734            ManagedPollOutcome::Idle => {}
4735        }
4736        if !self.queries.is_empty() {
4737            match self.poll_query_once().await? {
4738                ManagedPollOutcome::Handled => handled += 1,
4739                ManagedPollOutcome::Stop => return Ok(handled),
4740                ManagedPollOutcome::Idle => {}
4741            }
4742        }
4743        Ok(handled)
4744    }
4745
4746    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4747        let poll_request_id = unique_request_id("rust-workflow-poll");
4748        let response = self
4749            .retry_worker_operation(|| {
4750                self.client.poll_workflow_task_response_with_request_id(
4751                    &self.worker_id,
4752                    &self.task_queue,
4753                    self.poll_timeout,
4754                    &poll_request_id,
4755                    0,
4756                )
4757            })
4758            .await?;
4759        if response.outcome().should_stop() {
4760            return Ok(ManagedPollOutcome::Stop);
4761        }
4762        let Some(task) = response.task else {
4763            return Ok(ManagedPollOutcome::Idle);
4764        };
4765
4766        let task_id = task.task_id.clone();
4767        let attempt = task.workflow_task_attempt;
4768        let run_id = task.run_id.clone();
4769        let lease_owner = task
4770            .lease_owner
4771            .clone()
4772            .unwrap_or_else(|| self.worker_id.clone());
4773
4774        match self.execute_workflow_task(task) {
4775            Ok(commands) if commands.is_empty() => {
4776                // A replay can consume a recorded pending durable command
4777                // without producing a new command. The standalone protocol
4778                // acknowledges that state through the typed waiting outcome;
4779                // an empty completion is rejected by servers that require at
4780                // least one executable command.
4781                self.client
4782                    .fail_workflow_task_with_type(
4783                        &task_id,
4784                        &lease_owner,
4785                        attempt,
4786                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4787                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4788                    )
4789                    .await?;
4790            }
4791            Ok(commands) => {
4792                let completion = self
4793                    .client
4794                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4795                    .await;
4796                if let Err(error) = completion {
4797                    if !workflow_task_completion_is_terminal_timeout(
4798                        &error,
4799                        &task_id,
4800                        attempt,
4801                        run_id.as_deref(),
4802                    ) {
4803                        return Err(error);
4804                    }
4805                }
4806            }
4807            Err(error) => {
4808                self.client
4809                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4810                    .await?;
4811            }
4812        }
4813
4814        Ok(ManagedPollOutcome::Handled)
4815    }
4816
4817    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4818        while !stop.load(Ordering::SeqCst) {
4819            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4820                stop.store(true, Ordering::SeqCst);
4821                break;
4822            }
4823        }
4824
4825        Ok(())
4826    }
4827
4828    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4829        let poll_request_id = unique_request_id("rust-activity-poll");
4830        let response = self
4831            .retry_worker_operation(|| {
4832                self.client.poll_activity_task_response_with_request_id(
4833                    &self.worker_id,
4834                    &self.task_queue,
4835                    self.poll_timeout,
4836                    &poll_request_id,
4837                    0,
4838                )
4839            })
4840            .await?;
4841        if response.outcome().should_stop() {
4842            return Ok(ManagedPollOutcome::Stop);
4843        }
4844        let Some(task) = response.task else {
4845            return Ok(ManagedPollOutcome::Idle);
4846        };
4847
4848        let task_id = task.task_id.clone();
4849        let attempt_id = task
4850            .activity_attempt_id
4851            .clone()
4852            .or(task.attempt_id.clone())
4853            .unwrap_or_default();
4854        let lease_owner = task
4855            .lease_owner
4856            .clone()
4857            .unwrap_or_else(|| self.worker_id.clone());
4858        let codec = task.payload_codec.clone();
4859        let result = self.execute_activity_task(task).await;
4860        match result {
4861            Ok(value) => {
4862                let completion = self
4863                    .client
4864                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4865                    .await;
4866                if let Err(error) = completion {
4867                    if !activity_task_rejection_is_final(&error) {
4868                        return Err(error);
4869                    }
4870                }
4871            }
4872            Err(error) => {
4873                let failure = self
4874                    .client
4875                    .fail_activity_task(
4876                        &task_id,
4877                        &attempt_id,
4878                        &lease_owner,
4879                        error.to_string(),
4880                        false,
4881                    )
4882                    .await;
4883                if let Err(error) = failure {
4884                    if !activity_task_rejection_is_final(&error) {
4885                        return Err(error);
4886                    }
4887                }
4888            }
4889        }
4890
4891        Ok(ManagedPollOutcome::Handled)
4892    }
4893
4894    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4895        while !stop.load(Ordering::SeqCst) {
4896            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4897                stop.store(true, Ordering::SeqCst);
4898                break;
4899            }
4900        }
4901
4902        Ok(())
4903    }
4904
4905    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4906        let poll_request_id = unique_request_id("rust-query-poll");
4907        let response = self
4908            .retry_worker_operation(|| {
4909                self.client.poll_query_task_response_with_request_id(
4910                    &self.worker_id,
4911                    &self.task_queue,
4912                    self.poll_timeout,
4913                    &poll_request_id,
4914                    0,
4915                )
4916            })
4917            .await?;
4918        if response.outcome().should_stop() {
4919            return Ok(ManagedPollOutcome::Stop);
4920        }
4921        let Some(task) = response.task else {
4922            return Ok(ManagedPollOutcome::Idle);
4923        };
4924
4925        let query_task_id = task.query_task_id.clone();
4926        let attempt = task.query_task_attempt;
4927        let lease_owner = task
4928            .lease_owner
4929            .clone()
4930            .unwrap_or_else(|| self.worker_id.clone());
4931        let codec = task.payload_codec.clone();
4932
4933        match self.execute_query_task(task).await {
4934            Ok(value) => {
4935                let result_envelope = match encode_typed_envelope(&value, &codec) {
4936                    Ok(result_envelope) => result_envelope,
4937                    Err(error) => {
4938                        let failure = self
4939                            .client
4940                            .fail_query_task(
4941                                &query_task_id,
4942                                &lease_owner,
4943                                attempt,
4944                                error.to_string(),
4945                                "query_result_encode_failed",
4946                                "QueryResultEncodeFailed",
4947                            )
4948                            .await;
4949                        if let Err(error) = failure {
4950                            if !query_task_rejection_is_final(&error) {
4951                                return Err(error);
4952                            }
4953                        }
4954                        return Ok(ManagedPollOutcome::Handled);
4955                    }
4956                };
4957
4958                if let Err(error) = self
4959                    .client
4960                    .complete_query_task_with_envelope(
4961                        &query_task_id,
4962                        &lease_owner,
4963                        attempt,
4964                        value.clone().into_json()?,
4965                        result_envelope,
4966                    )
4967                    .await
4968                {
4969                    if !query_task_rejection_is_final(&error) {
4970                        return Err(error);
4971                    }
4972                }
4973            }
4974            Err(failure) => {
4975                let result = self
4976                    .client
4977                    .fail_query_task(
4978                        &query_task_id,
4979                        &lease_owner,
4980                        attempt,
4981                        failure.message,
4982                        failure.reason,
4983                        failure.failure_type,
4984                    )
4985                    .await;
4986                if let Err(error) = result {
4987                    if !query_task_rejection_is_final(&error) {
4988                        return Err(error);
4989                    }
4990                }
4991            }
4992        }
4993
4994        Ok(ManagedPollOutcome::Handled)
4995    }
4996
4997    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4998        while !stop.load(Ordering::SeqCst) {
4999            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
5000                stop.store(true, Ordering::SeqCst);
5001                break;
5002            }
5003        }
5004
5005        Ok(())
5006    }
5007
5008    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5009    where
5010        F: FnMut() -> Fut,
5011        Fut: Future<Output = Result<T>>,
5012    {
5013        let mut retries = 0;
5014
5015        loop {
5016            match operation().await {
5017                Err(error)
5018                    if worker_operation_is_retryable(&error)
5019                        && retries < self.retry_policy.max_retries =>
5020                {
5021                    retries += 1;
5022                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5023                }
5024                result => return result,
5025            }
5026        }
5027    }
5028
5029    async fn execute_query_task(
5030        &self,
5031        mut task: QueryTask,
5032    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5033        if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
5034            return Err(QueryTaskExecutionFailure::new(
5035                "query_payload_decode_failed",
5036                format!(
5037                    "cannot decode query payload with unsupported codec {:?}",
5038                    task.payload_codec
5039                ),
5040                "QueryPayloadDecodeFailed",
5041            ));
5042        }
5043
5044        if !self.workflows.contains_key(&task.workflow_type) {
5045            return Err(QueryTaskExecutionFailure::new(
5046                "query_workflow_type_not_registered",
5047                format!("no workflow registered for type {:?}", task.workflow_type),
5048                "WorkflowTypeNotRegistered",
5049            ));
5050        }
5051
5052        let Some(handlers) = self.queries.get(&task.workflow_type) else {
5053            return Err(QueryTaskExecutionFailure::new(
5054                "query_handler_unavailable",
5055                format!(
5056                    "query handlers are unavailable for workflow type {:?}",
5057                    task.workflow_type
5058                ),
5059                "QueryHandlerUnavailable",
5060            ));
5061        };
5062        let Some(query) = handlers.get(&task.query_name) else {
5063            return Err(QueryTaskExecutionFailure::new(
5064                "rejected_unknown_query",
5065                format!("unknown query {:?}", task.query_name),
5066                "QueryFailed",
5067            ));
5068        };
5069
5070        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5071            .map_err(|error| {
5072            QueryTaskExecutionFailure::new(
5073                "query_payload_decode_failed",
5074                format!("cannot decode query arguments: {error}"),
5075                "QueryPayloadDecodeFailed",
5076            )
5077        })?;
5078        let workflow_input_typed =
5079            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5080                .map_err(|error| {
5081                    QueryTaskExecutionFailure::new(
5082                        "query_workflow_state_unavailable",
5083                        format!("cannot decode workflow start input: {error}"),
5084                        "QueryWorkflowStateUnavailable",
5085                    )
5086                })?;
5087        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5088            QueryTaskExecutionFailure::new(
5089                "query_workflow_state_unavailable",
5090                format!("cannot project workflow start input: {error}"),
5091                "QueryWorkflowStateUnavailable",
5092            )
5093        })?;
5094        hydrate_query_history_from_export(&mut task).map_err(|error| {
5095            QueryTaskExecutionFailure::new(
5096                "query_workflow_state_unavailable",
5097                format!("cannot restore query history snapshot: {error}"),
5098                "QueryWorkflowStateUnavailable",
5099            )
5100        })?;
5101        enrich_query_history_from_export(&mut task).map_err(|error| {
5102            QueryTaskExecutionFailure::new(
5103                "query_workflow_state_unavailable",
5104                format!("cannot restore compact query history payloads: {error}"),
5105                "QueryWorkflowStateUnavailable",
5106            )
5107        })?;
5108        let signal_events = query_signal_events(&task).map_err(|error| {
5109            QueryTaskExecutionFailure::new(
5110                "query_workflow_state_unavailable",
5111                format!("cannot decode committed workflow signals: {error}"),
5112                "QueryWorkflowStateUnavailable",
5113            )
5114        })?;
5115        let history_events = Arc::new(std::mem::take(&mut task.history_events));
5116        let context = QueryContext {
5117            workflow_id: task.workflow_id,
5118            run_id: task.run_id,
5119            workflow_type: task.workflow_type.clone(),
5120            run_status: task.run_status,
5121            workflow_input,
5122            workflow_input_avro_value: workflow_input_typed.clone(),
5123            history_events: Arc::clone(&history_events),
5124            signal_events: Arc::new(signal_events),
5125        };
5126
5127        let future = match query {
5128            RegisteredQuery::Snapshot(handler) => handler(context, args),
5129            RegisteredQuery::Replayed {
5130                state_type,
5131                handler,
5132            } => {
5133                let workflow = self
5134                    .workflows
5135                    .get(&task.workflow_type)
5136                    .expect("workflow registration was checked above");
5137                if workflow.state_type != Some(*state_type) {
5138                    return Err(QueryTaskExecutionFailure::new(
5139                        "query_workflow_state_unavailable",
5140                        "replayed query state type does not match its workflow registration",
5141                        "QueryWorkflowStateUnavailable",
5142                    ));
5143                }
5144                let replay = workflow.replay.as_ref().ok_or_else(|| {
5145                    QueryTaskExecutionFailure::new(
5146                        "query_workflow_state_unavailable",
5147                        format!(
5148                            "workflow type {:?} is not registered for instance-state replay",
5149                            task.workflow_type
5150                        ),
5151                        "QueryWorkflowStateUnavailable",
5152                    )
5153                })?;
5154                let workflow_state = Arc::new(Mutex::new(
5155                    WorkflowState::new_with_identity(
5156                        history_events.as_ref().clone(),
5157                        context.workflow_id.clone(),
5158                        context.run_id.clone(),
5159                        self.task_queue.clone(),
5160                        task.payload_codec,
5161                        None,
5162                    )
5163                    .map_err(|error| {
5164                        QueryTaskExecutionFailure::new(
5165                            "query_workflow_state_unavailable",
5166                            format!("workflow replay failed before query: {error}"),
5167                            "QueryWorkflowStateUnavailable",
5168                        )
5169                    })?,
5170                ));
5171                let workflow_context = WorkflowContext {
5172                    state: workflow_state,
5173                };
5174                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5175                let mut cx = TaskContext::from_waker(noop_waker_ref());
5176                match invocation.future.as_mut().poll(&mut cx) {
5177                    Poll::Ready(Ok(_)) => {
5178                        workflow_context
5179                            .ensure_history_consumed()
5180                            .map_err(|error| {
5181                                QueryTaskExecutionFailure::new(
5182                                    "query_workflow_state_unavailable",
5183                                    format!("workflow replay failed before query: {error}"),
5184                                    "QueryWorkflowStateUnavailable",
5185                                )
5186                            })?;
5187                    }
5188                    Poll::Ready(Err(error)) => {
5189                        return Err(QueryTaskExecutionFailure::new(
5190                            "query_workflow_state_unavailable",
5191                            format!("workflow replay failed before query: {error}"),
5192                            "QueryWorkflowStateUnavailable",
5193                        ));
5194                    }
5195                    Poll::Pending => {
5196                        let commands = workflow_context.take_commands().map_err(|error| {
5197                            QueryTaskExecutionFailure::new(
5198                                "query_workflow_state_unavailable",
5199                                format!("workflow replay failed before query: {error}"),
5200                                "QueryWorkflowStateUnavailable",
5201                            )
5202                        })?;
5203                        if commands.is_empty()
5204                            && !workflow_context
5205                                .matched_recorded_pending()
5206                                .map_err(|error| {
5207                                    QueryTaskExecutionFailure::new(
5208                                        "query_workflow_state_unavailable",
5209                                        format!("workflow replay failed before query: {error}"),
5210                                        "QueryWorkflowStateUnavailable",
5211                                    )
5212                                })?
5213                        {
5214                            return Err(QueryTaskExecutionFailure::new(
5215                                "query_workflow_state_unavailable",
5216                                "workflow replay yielded without a durable command",
5217                                "QueryWorkflowStateUnavailable",
5218                            ));
5219                        }
5220                    }
5221                }
5222                let state = (invocation.snapshot)().map_err(|error| {
5223                    QueryTaskExecutionFailure::new(
5224                        "query_workflow_state_unavailable",
5225                        format!("cannot snapshot replayed workflow state: {error}"),
5226                        "QueryWorkflowStateUnavailable",
5227                    )
5228                })?;
5229                handler(context, state, args).map_err(|message| {
5230                    QueryTaskExecutionFailure::new(
5231                        "query_workflow_state_unavailable",
5232                        message,
5233                        "QueryWorkflowStateUnavailable",
5234                    )
5235                })?
5236            }
5237        };
5238
5239        future.await.map_err(|error| {
5240            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5241        })
5242    }
5243
5244    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5245        if let Some(update_id) = task
5246            .workflow_update_id
5247            .as_deref()
5248            .filter(|update_id| !update_id.is_empty())
5249        {
5250            return self.execute_update_task(&task, update_id);
5251        }
5252
5253        let workflow = self
5254            .workflows
5255            .get(&task.workflow_type)
5256            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5257        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5258        let resume_signal = decode_resume_signal(&task)?;
5259        let history_budget = WorkflowHistoryBudget {
5260            event_count: task
5261                .total_history_events
5262                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5263            size_bytes: task.history_size_bytes,
5264            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5265            pressure: task.history_budget_pressure.clone(),
5266        };
5267        let mut workflow_state = WorkflowState::new_with_identity(
5268            task.history_events,
5269            task.workflow_id,
5270            task.run_id,
5271            self.task_queue.clone(),
5272            task.payload_codec.clone(),
5273            resume_signal,
5274        )?;
5275        workflow_state.history_budget = history_budget;
5276        let state = Arc::new(Mutex::new(workflow_state));
5277        let ctx = WorkflowContext { state };
5278        let mut future = (workflow.execute)(ctx.clone(), input);
5279        let mut cx = TaskContext::from_waker(noop_waker_ref());
5280
5281        match future.as_mut().poll(&mut cx) {
5282            Poll::Ready(Ok(result)) => {
5283                ctx.ensure_history_consumed()?;
5284                let result = encode_typed_envelope(&result, &task.payload_codec)?;
5285                let mut commands = ctx.take_commands()?;
5286                commands.push(json!({
5287                    "type": "complete_workflow",
5288                    "result": result
5289                }));
5290                Ok(commands)
5291            }
5292            Poll::Ready(Err(error)) => {
5293                if let Error::ContinueAsNew(request) = error {
5294                    let mut commands = ctx.take_commands()?;
5295                    if let Some(command) = ctx.continue_as_new_command(request)? {
5296                        commands.push(command);
5297                    }
5298                    ctx.ensure_history_consumed()?;
5299                    return Ok(commands);
5300                }
5301                // A handler error must not hide a committed durable command that
5302                // upgraded workflow code no longer consumes.
5303                ctx.ensure_history_consumed()?;
5304                if workflow_task_integrity_error(&error) {
5305                    // Replay and protocol failures describe the workflow-task
5306                    // decision itself. Do not let commands queued earlier in
5307                    // this uncommitted decision escape alongside a terminal
5308                    // workflow failure.
5309                    return Err(error);
5310                }
5311                let mut commands = ctx.take_commands()?;
5312                commands.push(workflow_failure_command(&error));
5313                Ok(commands)
5314            }
5315            Poll::Pending => {
5316                let commands = ctx.take_commands()?;
5317                if commands.is_empty() && !ctx.matched_recorded_pending()? {
5318                    Err(Error::WorkflowYieldedWithoutCommand)
5319                } else {
5320                    Ok(commands)
5321                }
5322            }
5323        }
5324    }
5325
5326    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5327        if !self.workflows.contains_key(&task.workflow_type) {
5328            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5329        }
5330
5331        let accepted = task.history_events.iter().rev().find_map(|event| {
5332            (event.event_type == "UpdateAccepted"
5333                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5334            .then_some(&event.payload)
5335        });
5336        let update_name = accepted
5337            .and_then(|payload| payload.get("update_name"))
5338            .and_then(Value::as_str)
5339            .or(task.update_name.as_deref())
5340            .unwrap_or_default();
5341        let Some(handler) = self
5342            .updates
5343            .get(&task.workflow_type)
5344            .and_then(|handlers| handlers.get(update_name))
5345        else {
5346            return Ok(vec![json!({
5347                "type": "fail_update",
5348                "update_id": update_id,
5349                "message": format!(
5350                    "no update handler is registered for {}.{update_name}",
5351                    task.workflow_type
5352                ),
5353                "exception_type": "UnknownUpdate",
5354                "non_retryable": true,
5355            })]);
5356        };
5357        let arguments = accepted
5358            .and_then(|payload| payload.get("arguments"))
5359            .or(task.arguments.as_ref());
5360        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5361        let context = QueryContext {
5362            workflow_id: task.workflow_id.clone(),
5363            run_id: task.run_id.clone(),
5364            workflow_type: task.workflow_type.clone(),
5365            run_status: Some("running".to_string()),
5366            workflow_input: Value::Null,
5367            workflow_input_avro_value: AvroValue::Null,
5368            history_events: Arc::new(task.history_events.clone()),
5369            signal_events: Arc::new(Vec::new()),
5370        };
5371        let mut future = handler(context, arguments);
5372        let mut cx = TaskContext::from_waker(noop_waker_ref());
5373
5374        match future.as_mut().poll(&mut cx) {
5375            Poll::Ready(Ok(result)) => Ok(vec![json!({
5376                "type": "complete_update",
5377                "update_id": update_id,
5378                "result": encode_typed_envelope(&result, &task.payload_codec)?,
5379            })]),
5380            Poll::Ready(Err(error)) => Ok(vec![json!({
5381                "type": "fail_update",
5382                "update_id": update_id,
5383                "message": error.to_string(),
5384                "exception_type": "UpdateFailed",
5385                "non_retryable": true,
5386            })]),
5387            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5388        }
5389    }
5390
5391    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5392        let handler = self
5393            .activities
5394            .get(&task.activity_type)
5395            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5396        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5397        let attempt_id = task
5398            .activity_attempt_id
5399            .clone()
5400            .or(task.attempt_id.clone())
5401            .unwrap_or_default();
5402        let lease_owner = task
5403            .lease_owner
5404            .clone()
5405            .unwrap_or_else(|| self.worker_id.clone());
5406        let ctx = ActivityContext {
5407            client: self.client.clone(),
5408            task_id: task.task_id,
5409            activity_attempt_id: attempt_id,
5410            lease_owner,
5411            activity_type: task.activity_type,
5412            attempt_number: task.attempt_number,
5413            task_queue: self.task_queue.clone(),
5414            worker_id: self.worker_id.clone(),
5415        };
5416
5417        handler(ctx, args).await
5418    }
5419}
5420
5421fn poller_result(
5422    kind: &str,
5423    result: std::result::Result<Result<()>, tokio::task::JoinError>,
5424) -> Result<()> {
5425    match result {
5426        Ok(result) => result,
5427        Err(error) => Err(Error::WorkerLoop(format!(
5428            "{kind} poller join error: {error}"
5429        ))),
5430    }
5431}
5432
5433fn optional_poller_result(
5434    kind: &str,
5435    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5436) -> Result<()> {
5437    match result {
5438        Some(result) => poller_result(kind, result),
5439        None => Ok(()),
5440    }
5441}
5442
5443async fn join_pollers(
5444    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5445    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5446    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5447) -> Result<()> {
5448    let mut first_error = None;
5449
5450    if let Some(handle) = workflow_poller {
5451        if let Err(error) = poller_result("workflow", handle.await) {
5452            first_error.get_or_insert(error);
5453        }
5454    }
5455
5456    if let Some(handle) = activity_poller {
5457        if let Err(error) = poller_result("activity", handle.await) {
5458            first_error.get_or_insert(error);
5459        }
5460    }
5461
5462    if let Some(handle) = query_poller {
5463        if let Err(error) = poller_result("query", handle.await) {
5464            first_error.get_or_insert(error);
5465        }
5466    }
5467
5468    if let Some(error) = first_error {
5469        Err(error)
5470    } else {
5471        Ok(())
5472    }
5473}
5474
5475fn default_worker_id() -> String {
5476    let millis = SystemTime::now()
5477        .duration_since(UNIX_EPOCH)
5478        .unwrap_or_default()
5479        .as_millis();
5480    format!("rust-worker-{}-{millis}", std::process::id())
5481}
5482
5483fn percent_encode_path_segment(segment: &str) -> String {
5484    const HEX: &[u8; 16] = b"0123456789ABCDEF";
5485    let mut encoded = String::with_capacity(segment.len());
5486
5487    for byte in segment.bytes() {
5488        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5489            encoded.push(char::from(byte));
5490        } else {
5491            encoded.push('%');
5492            encoded.push(char::from(HEX[(byte >> 4) as usize]));
5493            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5494        }
5495    }
5496
5497    encoded
5498}
5499
5500fn unique_request_id(prefix: &str) -> String {
5501    let nanos = SystemTime::now()
5502        .duration_since(UNIX_EPOCH)
5503        .unwrap_or_default()
5504        .as_nanos();
5505    format!("{prefix}-{}-{nanos}", std::process::id())
5506}
5507
5508#[derive(Debug)]
5509struct QueryTaskExecutionFailure {
5510    reason: String,
5511    message: String,
5512    failure_type: String,
5513}
5514
5515impl QueryTaskExecutionFailure {
5516    fn new(
5517        reason: impl Into<String>,
5518        message: impl Into<String>,
5519        failure_type: impl Into<String>,
5520    ) -> Self {
5521        Self {
5522            reason: reason.into(),
5523            message: message.into(),
5524            failure_type: failure_type.into(),
5525        }
5526    }
5527}
5528
5529/// Typed local state owned by one deterministic workflow invocation.
5530///
5531/// Use [`WorkflowInstance::update`] for the same state transitions during
5532/// ordinary execution and replay. A replayed query receives a detached
5533/// immutable `Arc<S>` rather than this mutation-capable handle.
5534#[derive(Clone, Debug)]
5535pub struct WorkflowInstance<S> {
5536    state: Arc<Mutex<S>>,
5537}
5538
5539impl<S> WorkflowInstance<S> {
5540    fn new(state: S) -> Self {
5541        Self {
5542            state: Arc::new(Mutex::new(state)),
5543        }
5544    }
5545
5546    /// Read the current workflow-instance state without changing it.
5547    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5548        let state = self
5549            .state
5550            .lock()
5551            .map_err(|_| Error::WorkflowStatePoisoned)?;
5552        Ok(reader(&state))
5553    }
5554
5555    /// Apply one deterministic workflow-instance state transition.
5556    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5557        let mut state = self
5558            .state
5559            .lock()
5560            .map_err(|_| Error::WorkflowStatePoisoned)?;
5561        Ok(transition(&mut state))
5562    }
5563}
5564
5565impl<S: Clone> WorkflowInstance<S> {
5566    fn snapshot(&self) -> Result<S> {
5567        self.read(Clone::clone)
5568    }
5569}
5570
5571#[derive(Clone, Debug)]
5572pub struct WorkflowContext {
5573    state: Arc<Mutex<WorkflowState>>,
5574}
5575
5576impl WorkflowContext {
5577    /// Identity of the parent workflow currently being replayed.
5578    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5579        let state = self
5580            .state
5581            .lock()
5582            .map_err(|_| Error::WorkflowStatePoisoned)?;
5583        Ok(WorkflowIdentity {
5584            workflow_id: state.workflow_id.clone(),
5585            run_id: state.run_id.clone(),
5586        })
5587    }
5588
5589    /// Return the server-published history budget for this workflow task.
5590    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5591        let state = self
5592            .state
5593            .lock()
5594            .map_err(|_| Error::WorkflowStatePoisoned)?;
5595        Ok(state.history_budget.clone())
5596    }
5597
5598    /// Continue this workflow instance as a fresh run with replacement arguments.
5599    ///
5600    /// Return this value directly from the workflow handler. The worker converts
5601    /// it to the terminal protocol command only after replay has consumed every
5602    /// recorded durable command.
5603    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5604        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5605    }
5606
5607    /// Continue as new with optional workflow-type and task-queue overrides.
5608    pub fn continue_as_new_with_options<T: Serialize>(
5609        &self,
5610        options: ContinueAsNewOptions,
5611        args: T,
5612    ) -> Result<Value> {
5613        options.validate()?;
5614        Err(Error::ContinueAsNew(ContinueAsNewRequest {
5615            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5616            options,
5617        }))
5618    }
5619
5620    pub fn activity<T: Serialize>(
5621        &self,
5622        activity_type: impl Into<String>,
5623        args: T,
5624    ) -> ActivityCall {
5625        self.activity_with_options(activity_type, ActivityOptions::new(), args)
5626    }
5627
5628    pub fn activity_on_queue<T, Q>(
5629        &self,
5630        activity_type: impl Into<String>,
5631        task_queue: Option<Q>,
5632        args: T,
5633    ) -> ActivityCall
5634    where
5635        T: Serialize,
5636        Q: Into<String>,
5637    {
5638        let mut options = ActivityOptions::new();
5639        options.task_queue = task_queue.map(Into::into);
5640        self.activity_with_options(activity_type, options, args)
5641    }
5642
5643    /// Schedule one durable activity with retry, routing, and timeout options.
5644    ///
5645    /// Options are validated before the command is emitted. Once the command is
5646    /// recorded, replay consumes the same activity lifecycle at this command
5647    /// position and never emits a duplicate schedule.
5648    ///
5649    /// ```no_run
5650    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
5651    /// # use std::time::Duration;
5652    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
5653    /// let result = ctx
5654    ///     .activity_with_options(
5655    ///         "charge-card",
5656    ///         ActivityOptions::new()
5657    ///             .task_queue("payments")
5658    ///             .retry_policy(
5659    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
5660    ///                     Duration::from_secs(1),
5661    ///                     2,
5662    ///                     Some(Duration::from_secs(30)),
5663    ///                 ),
5664    ///             )
5665    ///             .start_to_close_timeout(Duration::from_secs(60))
5666    ///             .schedule_to_close_timeout(Duration::from_secs(180))
5667    ///             .heartbeat_timeout(Duration::from_secs(15)),
5668    ///         json!([{"order_id": "order-42"}]),
5669    ///     )
5670    ///     .await;
5671    /// match result {
5672    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
5673    ///         "reason": failure.reason,
5674    ///         "timeout_kind": failure.timeout_kind,
5675    ///     })),
5676    ///     other => other,
5677    /// }
5678    /// # }
5679    /// ```
5680    pub fn activity_with_options<T: Serialize>(
5681        &self,
5682        activity_type: impl Into<String>,
5683        options: ActivityOptions,
5684        args: T,
5685    ) -> ActivityCall {
5686        ActivityCall {
5687            ctx: self.clone(),
5688            activity_type: activity_type.into(),
5689            options,
5690            args: Some(AvroValue::from_serialize(&args)),
5691            scheduled: false,
5692        }
5693    }
5694
5695    pub async fn activity_avro_value<T: Serialize>(
5696        &self,
5697        activity_type: impl Into<String>,
5698        args: T,
5699    ) -> Result<AvroValue> {
5700        let mut call = self.activity(activity_type, args);
5701        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5702    }
5703
5704    pub async fn activity_avro_value_with_options<T: Serialize>(
5705        &self,
5706        activity_type: impl Into<String>,
5707        options: ActivityOptions,
5708        args: T,
5709    ) -> Result<AvroValue> {
5710        let mut call = self.activity_with_options(activity_type, options, args);
5711        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5712    }
5713
5714    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5715        SignalCall {
5716            ctx: self.clone(),
5717            signal_name: signal_name.into(),
5718            opened_wait: false,
5719            matched_pending: false,
5720        }
5721    }
5722
5723    pub async fn wait_signal_avro_value(
5724        &self,
5725        signal_name: impl Into<String>,
5726    ) -> Result<Vec<AvroValue>> {
5727        let mut call = self.wait_signal(signal_name);
5728        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5729    }
5730
5731    /// Wait for server-backed durable time without blocking the worker executor.
5732    ///
5733    /// Polling this future emits one `start_timer` command and yields. The
5734    /// server records the deadline, so neither worker nor server restarts reset
5735    /// the wait. Replay resolves the future only from a `TimerScheduled` and
5736    /// `TimerFired` pair at the same position in the shared durable-command
5737    /// stream, with matching sequence, timer identity, and delay. Sub-second
5738    /// durations round up because protocol deadlines use whole seconds.
5739    ///
5740    /// ```no_run
5741    /// # use durable_workflow::{json, Client, Worker};
5742    /// # use std::time::Duration;
5743    /// # fn configure(client: Client) {
5744    /// let mut worker = Worker::new(client, "rust-workers");
5745    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
5746    ///     ctx.sleep(Duration::from_secs(5)).await?;
5747    ///     Ok(json!({"status": "timer fired"}))
5748    /// });
5749    /// # }
5750    /// ```
5751    pub fn sleep(&self, duration: Duration) -> TimerCall {
5752        let delay_seconds = duration
5753            .as_secs()
5754            .checked_add(u64::from(duration.subsec_nanos() > 0));
5755        TimerCall {
5756            ctx: self.clone(),
5757            delay_seconds,
5758            scheduled: false,
5759            matched_pending: false,
5760        }
5761    }
5762
5763    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
5764    pub fn start_timer(&self, duration: Duration) -> TimerCall {
5765        self.sleep(duration)
5766    }
5767
5768    /// Evaluate a non-deterministic callback once and durably record its typed value.
5769    ///
5770    /// On replay the callback is not invoked: the value is decoded from the
5771    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
5772    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
5773    /// small values that must remain fixed for the lifetime of a workflow run.
5774    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5775    where
5776        T: Serialize + DeserializeOwned,
5777        F: FnOnce() -> T,
5778    {
5779        {
5780            let mut state = self
5781                .state
5782                .lock()
5783                .map_err(|_| Error::WorkflowStatePoisoned)?;
5784            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5785                return match recorded {
5786                    RecordedCommand::SideEffect { sequence, value } => {
5787                        state.command_cursor += 1;
5788                        value.deserialize().map_err(|error| {
5789                            Error::NonDeterministicReplay(ReplayFailure::new(
5790                                "side_effect_type_mismatch",
5791                                Some(sequence),
5792                                Some(std::any::type_name::<T>().to_string()),
5793                                Some(error.to_string()),
5794                                "recorded side-effect value is incompatible with the requested Rust type",
5795                            ))
5796                        })
5797                    }
5798                    other => Err(command_mismatch(&other, "side effect")),
5799                };
5800            }
5801        }
5802
5803        let value = callback();
5804        let avro_value = AvroValue::from_serialize(&value)?;
5805        let mut state = self
5806            .state
5807            .lock()
5808            .map_err(|_| Error::WorkflowStatePoisoned)?;
5809        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5810        state.commands.push(json!({
5811            "type": "record_side_effect",
5812            "result": result,
5813        }));
5814        Ok(value)
5815    }
5816
5817    /// Record or replay a lossless fixed Avro Value side effect.
5818    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5819    where
5820        F: FnOnce() -> AvroValue,
5821    {
5822        {
5823            let mut state = self
5824                .state
5825                .lock()
5826                .map_err(|_| Error::WorkflowStatePoisoned)?;
5827            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5828                return match recorded {
5829                    RecordedCommand::SideEffect { value, .. } => {
5830                        state.command_cursor += 1;
5831                        Ok(value)
5832                    }
5833                    other => Err(command_mismatch(&other, "side effect")),
5834                };
5835            }
5836        }
5837
5838        let value = callback();
5839        let mut state = self
5840            .state
5841            .lock()
5842            .map_err(|_| Error::WorkflowStatePoisoned)?;
5843        let result = encode_typed_envelope(&value, &state.payload_codec)?;
5844        state.commands.push(json!({
5845            "type": "record_side_effect",
5846            "result": result,
5847        }));
5848        Ok(value)
5849    }
5850
5851    /// Record a UUIDv4 once and return the same UUID on every replay.
5852    pub fn uuid_v4(&self) -> Result<Uuid> {
5853        self.side_effect(Uuid::new_v4)
5854    }
5855
5856    /// Select the newest supported version for a change, or replay the version
5857    /// already committed for that stable change ID.
5858    pub fn get_version(
5859        &self,
5860        change_id: impl Into<String>,
5861        min_supported: i32,
5862        max_supported: i32,
5863    ) -> Result<i32> {
5864        let change_id = change_id.into();
5865        if change_id.trim().is_empty() {
5866            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5867                "version_change_id_invalid",
5868                None,
5869                Some("non-empty change ID".to_string()),
5870                Some(change_id),
5871                "version markers require a stable non-empty change ID",
5872            )));
5873        }
5874        if min_supported > max_supported {
5875            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5876                "version_range_invalid",
5877                None,
5878                Some("min_supported <= max_supported".to_string()),
5879                Some(format!("{min_supported}..={max_supported}")),
5880                "version marker supported range is invalid",
5881            )));
5882        }
5883
5884        let mut state = self
5885            .state
5886            .lock()
5887            .map_err(|_| Error::WorkflowStatePoisoned)?;
5888        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5889            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5890            return Ok(version);
5891        }
5892
5893        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5894            return match recorded {
5895                RecordedCommand::VersionMarker {
5896                    sequence,
5897                    change_id: recorded_change_id,
5898                    version,
5899                    ..
5900                } => {
5901                    if recorded_change_id != change_id {
5902                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5903                            "version_change_id_mismatch",
5904                            Some(sequence),
5905                            Some(recorded_change_id),
5906                            Some(change_id),
5907                            "recorded version marker change ID differs from current workflow code",
5908                        )));
5909                    }
5910                    ensure_version_supported(
5911                        &change_id,
5912                        version,
5913                        min_supported,
5914                        max_supported,
5915                        sequence,
5916                    )?;
5917                    state.command_cursor += 1;
5918                    state.version_markers.insert(change_id, (version, sequence));
5919                    Ok(version)
5920                }
5921                other => Err(command_mismatch(
5922                    &other,
5923                    format!("version marker:{change_id}"),
5924                )),
5925            };
5926        }
5927
5928        let version = max_supported;
5929        state.commands.push(json!({
5930            "type": "record_version_marker",
5931            "change_id": change_id,
5932            "version": version,
5933            "min_supported": min_supported,
5934            "max_supported": max_supported,
5935        }));
5936        // Sequence numbers are assigned by the server. Zero identifies a marker
5937        // selected in this uncommitted decision batch for duplicate-call checks.
5938        state.version_markers.insert(change_id, (version, 0));
5939        Ok(version)
5940    }
5941
5942    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
5943    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5944        Ok(self.get_version(change_id, -1, 1)? == 1)
5945    }
5946
5947    /// Keep a patch marker in history after the legacy branch has been removed.
5948    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5949        self.get_version(change_id, -1, 1).map(|_| ())
5950    }
5951
5952    /// Start a named durable child on an explicit queue and await its result.
5953    ///
5954    /// The command is recorded in the parent's sequence-ordered durable command
5955    /// stream. Replay keeps a scheduled child pending without emitting another
5956    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
5957    /// values preserve the history payload codec and include both sides of the
5958    /// durable relationship; failures are returned as
5959    /// [`Error::ChildWorkflowFailed`].
5960    ///
5961    /// ```no_run
5962    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
5963    /// # fn configure(client: Client) {
5964    /// let mut worker = Worker::new(client, "parent-workers");
5965    /// worker.register_workflow("order-parent", |ctx, _input| async move {
5966    ///     let child = ctx
5967    ///         .start_child_workflow(
5968    ///             "fulfil-order",
5969    ///             ChildWorkflowOptions::new("fulfilment-workers")
5970    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
5971    ///             json!([{"order_id": "order-42"}]),
5972    ///         )
5973    ///         .await?;
5974    ///     Ok(child.result)
5975    /// });
5976    /// # }
5977    /// ```
5978    pub fn start_child_workflow<T: Serialize>(
5979        &self,
5980        workflow_type: impl Into<String>,
5981        options: ChildWorkflowOptions,
5982        args: T,
5983    ) -> ChildWorkflowCall {
5984        ChildWorkflowCall {
5985            ctx: self.clone(),
5986            workflow_type: workflow_type.into(),
5987            options,
5988            args: Some(AvroValue::from_serialize(&args)),
5989            scheduled: false,
5990            matched_pending: false,
5991        }
5992    }
5993
5994    pub async fn start_child_workflow_avro_value<T: Serialize>(
5995        &self,
5996        workflow_type: impl Into<String>,
5997        options: ChildWorkflowOptions,
5998        args: T,
5999    ) -> Result<ChildWorkflowAvroResult> {
6000        let mut call = self.start_child_workflow(workflow_type, options, args);
6001        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6002    }
6003
6004    fn take_commands(&self) -> Result<Vec<Value>> {
6005        let mut state = self
6006            .state
6007            .lock()
6008            .map_err(|_| Error::WorkflowStatePoisoned)?;
6009        Ok(std::mem::take(&mut state.commands))
6010    }
6011
6012    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6013        let mut state = self
6014            .state
6015            .lock()
6016            .map_err(|_| Error::WorkflowStatePoisoned)?;
6017
6018        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6019            return Err(command_mismatch(&recorded, "continue as new"));
6020        }
6021        if state.recorded_continue_as_new_sequence.is_some() {
6022            state.continue_as_new_consumed = true;
6023            return Ok(None);
6024        }
6025
6026        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6027        let mut command = serde_json::Map::from_iter([
6028            ("type".to_string(), json!("continue_as_new")),
6029            ("arguments".to_string(), arguments),
6030            ("queue".to_string(), json!(state.task_queue.clone())),
6031        ]);
6032        if let Some(workflow_type) = request.options.workflow_type {
6033            command.insert("workflow_type".to_string(), json!(workflow_type));
6034        }
6035        if let Some(task_queue) = request.options.task_queue {
6036            command.insert("queue".to_string(), json!(task_queue));
6037        }
6038        Ok(Some(Value::Object(command)))
6039    }
6040
6041    fn matched_recorded_pending(&self) -> Result<bool> {
6042        let state = self
6043            .state
6044            .lock()
6045            .map_err(|_| Error::WorkflowStatePoisoned)?;
6046        Ok(state.matched_recorded_pending)
6047    }
6048
6049    fn ensure_history_consumed(&self) -> Result<()> {
6050        let state = self
6051            .state
6052            .lock()
6053            .map_err(|_| Error::WorkflowStatePoisoned)?;
6054        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6055            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6056                "recorded_commands_unconsumed",
6057                Some(command.sequence()),
6058                Some(command.shape().to_string()),
6059                Some("workflow completion".to_string()),
6060                "workflow completed before consuming all recorded durable commands",
6061            )));
6062        }
6063        if let Some(sequence) = state
6064            .recorded_continue_as_new_sequence
6065            .filter(|_| !state.continue_as_new_consumed)
6066        {
6067            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6068                "recorded_continue_as_new_unconsumed",
6069                Some(sequence),
6070                Some("continue as new".to_string()),
6071                Some("workflow completion".to_string()),
6072                "workflow completed without consuming its recorded continue-as-new transition",
6073            )));
6074        }
6075        Ok(())
6076    }
6077}
6078
6079#[derive(Debug)]
6080struct WorkflowState {
6081    workflow_id: Option<String>,
6082    run_id: Option<String>,
6083    task_queue: String,
6084    payload_codec: String,
6085    history_budget: WorkflowHistoryBudget,
6086    resume_signal: Option<ResumeSignal>,
6087    recorded_commands: Vec<RecordedCommand>,
6088    recorded_continue_as_new_sequence: Option<u64>,
6089    continue_as_new_consumed: bool,
6090    command_cursor: usize,
6091    matched_recorded_pending: bool,
6092    version_markers: HashMap<String, (i32, u64)>,
6093    commands: Vec<Value>,
6094}
6095
6096impl WorkflowState {
6097    #[cfg(test)]
6098    fn new(
6099        history: Vec<HistoryEvent>,
6100        task_queue: String,
6101        payload_codec: String,
6102        resume_signal: Option<ResumeSignal>,
6103    ) -> Result<Self> {
6104        Self::new_with_identity(
6105            history,
6106            None,
6107            None,
6108            task_queue,
6109            payload_codec,
6110            resume_signal,
6111        )
6112    }
6113
6114    fn new_with_identity(
6115        history: Vec<HistoryEvent>,
6116        workflow_id: Option<String>,
6117        run_id: Option<String>,
6118        task_queue: String,
6119        payload_codec: String,
6120        resume_signal: Option<ResumeSignal>,
6121    ) -> Result<Self> {
6122        let recorded_commands = recorded_commands(
6123            &history,
6124            &payload_codec,
6125            WorkflowIdentity {
6126                workflow_id: workflow_id.clone(),
6127                run_id: run_id.clone(),
6128            },
6129        )?;
6130        let recorded_continue_as_new = history
6131            .iter()
6132            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6133            .collect::<Vec<_>>();
6134        if recorded_continue_as_new.len() > 1 {
6135            return Err(invalid_recorded_history(
6136                "duplicate_continue_as_new_transition",
6137                recorded_continue_as_new
6138                    .last()
6139                    .and_then(|event| durable_event_sequence(event))
6140                    .unwrap_or(0),
6141                "one WorkflowContinuedAsNew event",
6142                &format!(
6143                    "{} WorkflowContinuedAsNew events",
6144                    recorded_continue_as_new.len()
6145                ),
6146                "workflow history records one continue-as-new transition more than once",
6147            ));
6148        }
6149        let recorded_continue_as_new_sequence = recorded_continue_as_new
6150            .first()
6151            .map(|event| {
6152                durable_event_sequence(event).ok_or_else(|| {
6153                    Error::NonDeterministicReplay(ReplayFailure::new(
6154                        "continue_as_new_sequence_missing",
6155                        None,
6156                        Some("recorded transition sequence".to_string()),
6157                        Some("missing sequence".to_string()),
6158                        "WorkflowContinuedAsNew history is missing its recorded sequence",
6159                    ))
6160                })
6161            })
6162            .transpose()?;
6163        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6164        Ok(Self {
6165            workflow_id,
6166            run_id,
6167            task_queue,
6168            payload_codec,
6169            history_budget: WorkflowHistoryBudget {
6170                event_count,
6171                ..WorkflowHistoryBudget::default()
6172            },
6173            resume_signal,
6174            recorded_commands,
6175            recorded_continue_as_new_sequence,
6176            continue_as_new_consumed: false,
6177            command_cursor: 0,
6178            matched_recorded_pending: false,
6179            version_markers: HashMap::new(),
6180            commands: Vec::new(),
6181        })
6182    }
6183}
6184
6185#[derive(Clone, Debug)]
6186enum RecordedCommand {
6187    Activity {
6188        sequence: u64,
6189        activity_type: Option<String>,
6190        options: Option<RecordedActivityOptions>,
6191        outcome: Option<ActivityOutcome>,
6192    },
6193    Timer {
6194        sequence: u64,
6195        delay_seconds: u64,
6196        fired: bool,
6197    },
6198    ChildWorkflow {
6199        sequence: u64,
6200        workflow_type: Option<String>,
6201        outcome: Option<ChildWorkflowOutcome>,
6202    },
6203    SignalWait {
6204        sequence: u64,
6205        signal_name: String,
6206        value: Option<Vec<AvroValue>>,
6207    },
6208    SideEffect {
6209        sequence: u64,
6210        value: AvroValue,
6211    },
6212    VersionMarker {
6213        sequence: u64,
6214        change_id: String,
6215        version: i32,
6216    },
6217}
6218
6219#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6220struct RecordedActivityOptions {
6221    task_queue: RecordedSnapshotValue<Option<String>>,
6222    execution_mode: RecordedSnapshotValue<Option<String>>,
6223    retry_policy: ActivityRetrySnapshot,
6224}
6225
6226#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6227enum RecordedSnapshotValue<T> {
6228    /// Older history did not persist this field, so it cannot constrain replay.
6229    Unknown,
6230    Known(T),
6231}
6232
6233impl<T: PartialEq> RecordedSnapshotValue<T> {
6234    fn matches_current(&self, current: &Self) -> bool {
6235        match self {
6236            Self::Unknown => true,
6237            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6238        }
6239    }
6240}
6241
6242#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6243struct ActivityRetrySnapshot {
6244    snapshot_version: RecordedSnapshotValue<Option<u64>>,
6245    max_attempts: RecordedSnapshotValue<Option<u64>>,
6246    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6247    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6248    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6249    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6250    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6251    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6252}
6253
6254impl ActivityRetrySnapshot {
6255    fn matches_current(&self, current: &Self) -> bool {
6256        self.snapshot_version
6257            .matches_current(&current.snapshot_version)
6258            && self.max_attempts.matches_current(&current.max_attempts)
6259            && self
6260                .backoff_seconds
6261                .matches_current(&current.backoff_seconds)
6262            && self
6263                .start_to_close_timeout
6264                .matches_current(&current.start_to_close_timeout)
6265            && self
6266                .schedule_to_start_timeout
6267                .matches_current(&current.schedule_to_start_timeout)
6268            && self
6269                .schedule_to_close_timeout
6270                .matches_current(&current.schedule_to_close_timeout)
6271            && self
6272                .heartbeat_timeout
6273                .matches_current(&current.heartbeat_timeout)
6274            && self
6275                .non_retryable_error_types
6276                .matches_current(&current.non_retryable_error_types)
6277    }
6278}
6279
6280fn recorded_optional_u64(
6281    object: Option<&serde_json::Map<String, Value>>,
6282    field: &str,
6283) -> RecordedSnapshotValue<Option<u64>> {
6284    match object.and_then(|object| object.get(field)) {
6285        None => RecordedSnapshotValue::Unknown,
6286        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6287        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6288    }
6289}
6290
6291fn recorded_optional_string(
6292    object: &serde_json::Map<String, Value>,
6293    field: &str,
6294) -> RecordedSnapshotValue<Option<String>> {
6295    match object.get(field) {
6296        None => RecordedSnapshotValue::Unknown,
6297        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6298        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6299    }
6300}
6301
6302fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6303    let policy = policy.and_then(Value::as_object);
6304    let backoff_seconds = policy
6305        .and_then(|policy| policy.get("backoff_seconds"))
6306        .and_then(Value::as_array)
6307        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6308        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6309    let mut non_retryable_error_types = Vec::new();
6310    for error_type in policy
6311        .and_then(|policy| policy.get("non_retryable_error_types"))
6312        .and_then(Value::as_array)
6313        .into_iter()
6314        .flatten()
6315        .filter_map(Value::as_str)
6316        .map(str::trim)
6317        .filter(|error_type| !error_type.is_empty())
6318    {
6319        if !non_retryable_error_types
6320            .iter()
6321            .any(|recorded| recorded == error_type)
6322        {
6323            non_retryable_error_types.push(error_type.to_string());
6324        }
6325    }
6326
6327    ActivityRetrySnapshot {
6328        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6329        max_attempts: recorded_optional_u64(policy, "max_attempts"),
6330        backoff_seconds,
6331        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6332        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6333        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6334        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6335        non_retryable_error_types: if policy
6336            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6337        {
6338            RecordedSnapshotValue::Known(non_retryable_error_types)
6339        } else {
6340            RecordedSnapshotValue::Unknown
6341        },
6342    }
6343}
6344
6345fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6346    let policy = options.retry_policy.as_ref();
6347    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6348        Some(Value::Null) => None,
6349        Some(value) => value_as_u64(value),
6350        None => Some(1),
6351    };
6352    let backoff_seconds = policy
6353        .and_then(|policy| policy.get("backoff_seconds"))
6354        .and_then(Value::as_array)
6355        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6356        .unwrap_or_default();
6357    let non_retryable_error_types = policy
6358        .and_then(|policy| policy.get("non_retryable_error_types"))
6359        .and_then(Value::as_array)
6360        .into_iter()
6361        .flatten()
6362        .filter_map(Value::as_str)
6363        .map(str::to_string)
6364        .collect();
6365
6366    ActivityRetrySnapshot {
6367        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6368        max_attempts: RecordedSnapshotValue::Known(max_attempts),
6369        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6370        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6371        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6372        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6373        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6374        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6375    }
6376}
6377
6378fn activity_options_description(options: &RecordedActivityOptions) -> String {
6379    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6380}
6381
6382impl RecordedCommand {
6383    fn sequence(&self) -> u64 {
6384        match self {
6385            Self::Activity { sequence, .. }
6386            | Self::Timer { sequence, .. }
6387            | Self::ChildWorkflow { sequence, .. }
6388            | Self::SignalWait { sequence, .. }
6389            | Self::SideEffect { sequence, .. }
6390            | Self::VersionMarker { sequence, .. } => *sequence,
6391        }
6392    }
6393
6394    fn shape(&self) -> &'static str {
6395        match self {
6396            Self::Activity { .. } => "activity",
6397            Self::Timer { .. } => "timer",
6398            Self::ChildWorkflow { .. } => "child workflow",
6399            Self::SignalWait { .. } => "signal wait",
6400            Self::SideEffect { .. } => "side effect",
6401            Self::VersionMarker { .. } => "version marker",
6402        }
6403    }
6404}
6405
6406fn ensure_version_supported(
6407    change_id: &str,
6408    version: i32,
6409    min_supported: i32,
6410    max_supported: i32,
6411    sequence: u64,
6412) -> Result<()> {
6413    if (min_supported..=max_supported).contains(&version) {
6414        return Ok(());
6415    }
6416    Err(Error::NonDeterministicReplay(ReplayFailure::new(
6417        "version_marker_incompatible_range",
6418        (sequence != 0).then_some(sequence),
6419        Some(format!("{min_supported}..={max_supported}")),
6420        Some(format!("{change_id}:{version}")),
6421        "recorded workflow version is outside the range supported by current code",
6422    )))
6423}
6424
6425#[derive(Clone, Debug)]
6426struct ResumeSignal {
6427    signal_name: String,
6428    arguments: Vec<AvroValue>,
6429}
6430
6431pub struct ActivityCall {
6432    ctx: WorkflowContext,
6433    activity_type: String,
6434    options: ActivityOptions,
6435    args: Option<Result<AvroValue>>,
6436    scheduled: bool,
6437}
6438
6439impl ActivityCall {
6440    fn poll_avro_value(
6441        mut self: Pin<&mut Self>,
6442        _cx: &mut TaskContext<'_>,
6443    ) -> Poll<Result<AvroValue>> {
6444        let ctx = self.ctx.clone();
6445        let mut state = match ctx.state.lock() {
6446            Ok(state) => state,
6447            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6448        };
6449
6450        if self.scheduled {
6451            return Poll::Pending;
6452        }
6453
6454        let options = match self.options.validate() {
6455            Ok(options) => options,
6456            Err(error) => {
6457                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6458            }
6459        };
6460        let task_queue = options
6461            .task_queue
6462            .clone()
6463            .unwrap_or_else(|| state.task_queue.clone());
6464        let current_recorded_options = RecordedActivityOptions {
6465            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6466            // Rust schedules ordinary durable activities. The server records a
6467            // non-null mode only for a specialized execution primitive.
6468            execution_mode: RecordedSnapshotValue::Known(None),
6469            retry_policy: current_activity_retry_snapshot(&options),
6470        };
6471
6472        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6473            let sequence = recorded.sequence();
6474            match recorded {
6475                RecordedCommand::Activity {
6476                    activity_type,
6477                    options: recorded_options,
6478                    outcome,
6479                    ..
6480                } => {
6481                    if let Some(recorded_type) = activity_type {
6482                        if recorded_type != self.activity_type {
6483                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6484                                ReplayFailure::new(
6485                                    "recorded_command_detail_mismatch",
6486                                    Some(sequence),
6487                                    Some(format!("activity:{recorded_type}")),
6488                                    Some(format!("activity:{}", self.activity_type)),
6489                                    "recorded activity type differs from the current workflow command",
6490                                ),
6491                            )));
6492                        }
6493                    }
6494                    if let Some(recorded_options) = recorded_options {
6495                        if !recorded_options
6496                            .task_queue
6497                            .matches_current(&current_recorded_options.task_queue)
6498                        {
6499                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6500                                ReplayFailure::new(
6501                                    "activity_task_queue_mismatch",
6502                                    Some(sequence),
6503                                    Some(activity_options_description(&recorded_options)),
6504                                    Some(activity_options_description(&current_recorded_options)),
6505                                    "recorded activity task queue differs from the current workflow command",
6506                                ),
6507                            )));
6508                        }
6509                        if !recorded_options
6510                            .execution_mode
6511                            .matches_current(&current_recorded_options.execution_mode)
6512                        {
6513                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6514                                ReplayFailure::new(
6515                                    "activity_execution_mode_mismatch",
6516                                    Some(sequence),
6517                                    Some(activity_options_description(&recorded_options)),
6518                                    Some(activity_options_description(&current_recorded_options)),
6519                                    "recorded activity execution mode differs from the current workflow command",
6520                                ),
6521                            )));
6522                        }
6523                        if !recorded_options
6524                            .retry_policy
6525                            .matches_current(&current_recorded_options.retry_policy)
6526                        {
6527                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6528                                ReplayFailure::new(
6529                                    "activity_retry_policy_mismatch",
6530                                    Some(sequence),
6531                                    Some(activity_options_description(&recorded_options)),
6532                                    Some(activity_options_description(&current_recorded_options)),
6533                                    "recorded activity retry policy differs from the current workflow command",
6534                                ),
6535                            )));
6536                        }
6537                    }
6538                    state.command_cursor += 1;
6539                    if let Some(outcome) = outcome {
6540                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6541                    }
6542                    state.matched_recorded_pending = true;
6543                    self.scheduled = true;
6544                    return Poll::Pending;
6545                }
6546                other => {
6547                    return Poll::Ready(Err(command_mismatch(
6548                        &other,
6549                        format!("activity:{}", self.activity_type),
6550                    )));
6551                }
6552            }
6553        }
6554
6555        if !self.scheduled {
6556            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6557                Ok(args) => args,
6558                Err(error) => return Poll::Ready(Err(error)),
6559            };
6560            let arguments = normalize_avro_arguments(args);
6561            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6562                Ok(envelope) => envelope,
6563                Err(error) => return Poll::Ready(Err(error)),
6564            };
6565
6566            let mut command = serde_json::Map::from_iter([
6567                ("type".to_string(), json!("schedule_activity")),
6568                (
6569                    "activity_type".to_string(),
6570                    json!(self.activity_type.clone()),
6571                ),
6572                ("queue".to_string(), json!(task_queue)),
6573                ("arguments".to_string(), envelope),
6574            ]);
6575            for (field, value) in [
6576                ("start_to_close_timeout", options.start_to_close_timeout),
6577                (
6578                    "schedule_to_start_timeout",
6579                    options.schedule_to_start_timeout,
6580                ),
6581                (
6582                    "schedule_to_close_timeout",
6583                    options.schedule_to_close_timeout,
6584                ),
6585                ("heartbeat_timeout", options.heartbeat_timeout),
6586            ] {
6587                if let Some(value) = value {
6588                    command.insert(field.to_string(), json!(value));
6589                }
6590            }
6591            if let Some(retry_policy) = options.retry_policy {
6592                command.insert("retry_policy".to_string(), retry_policy);
6593            }
6594            state.commands.push(Value::Object(command));
6595            self.scheduled = true;
6596        }
6597
6598        Poll::Pending
6599    }
6600}
6601
6602impl Future for ActivityCall {
6603    type Output = Result<Value>;
6604
6605    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6606        match self.poll_avro_value(cx) {
6607            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6608            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6609            Poll::Pending => Poll::Pending,
6610        }
6611    }
6612}
6613
6614/// Future returned by [`WorkflowContext::sleep`].
6615pub struct TimerCall {
6616    ctx: WorkflowContext,
6617    delay_seconds: Option<u64>,
6618    scheduled: bool,
6619    matched_pending: bool,
6620}
6621
6622impl Future for TimerCall {
6623    type Output = Result<()>;
6624
6625    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6626        if self.matched_pending {
6627            return Poll::Pending;
6628        }
6629
6630        let ctx = self.ctx.clone();
6631        let Some(requested_delay) = self.delay_seconds else {
6632            return Poll::Ready(Err(Error::TimerDurationOverflow));
6633        };
6634        let mut state = match ctx.state.lock() {
6635            Ok(state) => state,
6636            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6637        };
6638
6639        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6640            match recorded {
6641                RecordedCommand::Timer {
6642                    sequence,
6643                    delay_seconds,
6644                    fired,
6645                    ..
6646                } => {
6647                    if delay_seconds != requested_delay {
6648                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6649                            ReplayFailure::new(
6650                                "timer_delay_mismatch",
6651                                Some(sequence),
6652                                Some(format!("timer:{delay_seconds}s")),
6653                                Some(format!("timer:{requested_delay}s")),
6654                                "recorded timer delay differs from the current workflow command",
6655                            ),
6656                        )));
6657                    }
6658                    state.command_cursor += 1;
6659                    if fired {
6660                        return Poll::Ready(Ok(()));
6661                    }
6662                    state.matched_recorded_pending = true;
6663                    self.scheduled = true;
6664                    self.matched_pending = true;
6665                    return Poll::Pending;
6666                }
6667                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6668            }
6669        }
6670
6671        if !self.scheduled {
6672            state.commands.push(json!({
6673                "type": "start_timer",
6674                "delay_seconds": requested_delay,
6675            }));
6676            self.scheduled = true;
6677        }
6678
6679        Poll::Pending
6680    }
6681}
6682
6683/// Future returned by [`WorkflowContext::start_child_workflow`].
6684pub struct ChildWorkflowCall {
6685    ctx: WorkflowContext,
6686    workflow_type: String,
6687    options: ChildWorkflowOptions,
6688    args: Option<Result<AvroValue>>,
6689    scheduled: bool,
6690    matched_pending: bool,
6691}
6692
6693impl ChildWorkflowCall {
6694    fn poll_avro_value(
6695        mut self: Pin<&mut Self>,
6696        _cx: &mut TaskContext<'_>,
6697    ) -> Poll<Result<ChildWorkflowAvroResult>> {
6698        if self.matched_pending {
6699            return Poll::Pending;
6700        }
6701
6702        let ctx = self.ctx.clone();
6703        let mut state = match ctx.state.lock() {
6704            Ok(state) => state,
6705            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6706        };
6707
6708        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6709            let sequence = recorded.sequence();
6710            match recorded {
6711                RecordedCommand::ChildWorkflow {
6712                    workflow_type,
6713                    outcome,
6714                    ..
6715                } => {
6716                    if let Some(recorded_type) = workflow_type {
6717                        if recorded_type != self.workflow_type {
6718                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6719                                ReplayFailure::new(
6720                                    "recorded_command_detail_mismatch",
6721                                    Some(sequence),
6722                                    Some(format!("child workflow:{recorded_type}")),
6723                                    Some(format!("child workflow:{}", self.workflow_type)),
6724                                    "recorded child workflow type differs from the current workflow command",
6725                                ),
6726                            )));
6727                        }
6728                    }
6729                    state.command_cursor += 1;
6730                    if let Some(outcome) = outcome {
6731                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6732                    }
6733                    state.matched_recorded_pending = true;
6734                    self.scheduled = true;
6735                    self.matched_pending = true;
6736                    return Poll::Pending;
6737                }
6738                other => {
6739                    return Poll::Ready(Err(command_mismatch(
6740                        &other,
6741                        format!("child workflow:{}", self.workflow_type),
6742                    )));
6743                }
6744            }
6745        }
6746
6747        if !self.scheduled {
6748            if self.options.task_queue.trim().is_empty() {
6749                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6750                    "task_queue must not be empty".to_string(),
6751                )));
6752            }
6753            for (name, value) in [
6754                (
6755                    "execution_timeout_seconds",
6756                    self.options.execution_timeout_seconds,
6757                ),
6758                ("run_timeout_seconds", self.options.run_timeout_seconds),
6759            ] {
6760                if value == Some(0) {
6761                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6762                        "{name} must be at least 1"
6763                    ))));
6764                }
6765            }
6766
6767            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6768                Ok(args) => args,
6769                Err(error) => return Poll::Ready(Err(error)),
6770            };
6771            let arguments = match encode_typed_envelope(
6772                &normalize_avro_arguments(args),
6773                &state.payload_codec,
6774            ) {
6775                Ok(arguments) => arguments,
6776                Err(error) => return Poll::Ready(Err(error)),
6777            };
6778            let mut command = json!({
6779                "type": "start_child_workflow",
6780                "workflow_type": self.workflow_type,
6781                "queue": self.options.task_queue,
6782                "parent_close_policy": self.options.parent_close_policy.as_str(),
6783                "arguments": arguments,
6784            });
6785            let object = command
6786                .as_object_mut()
6787                .expect("child workflow command is always an object");
6788            if let Some(policy) = &self.options.retry_policy {
6789                let mut retry_policy = serde_json::Map::new();
6790                if let Some(max_attempts) = policy.max_attempts {
6791                    if max_attempts == 0 {
6792                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6793                            "retry_policy.max_attempts must be at least 1".to_string(),
6794                        )));
6795                    }
6796                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6797                }
6798                if !policy.backoff_seconds.is_empty() {
6799                    retry_policy
6800                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6801                }
6802                if !policy.non_retryable_error_types.is_empty() {
6803                    retry_policy.insert(
6804                        "non_retryable_error_types".to_string(),
6805                        json!(policy.non_retryable_error_types),
6806                    );
6807                }
6808                if retry_policy.is_empty() {
6809                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6810                        "retry_policy must configure at least one field".to_string(),
6811                    )));
6812                }
6813                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6814            }
6815            if let Some(seconds) = self.options.execution_timeout_seconds {
6816                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6817            }
6818            if let Some(seconds) = self.options.run_timeout_seconds {
6819                object.insert("run_timeout_seconds".to_string(), json!(seconds));
6820            }
6821            state.commands.push(command);
6822            self.scheduled = true;
6823        }
6824
6825        Poll::Pending
6826    }
6827}
6828
6829impl Future for ChildWorkflowCall {
6830    type Output = Result<ChildWorkflowResult>;
6831
6832    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6833        match self.poll_avro_value(cx) {
6834            Poll::Ready(Ok(result)) => match result.result.into_json() {
6835                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6836                    parent: result.parent,
6837                    child: result.child,
6838                    child_workflow_type: result.child_workflow_type,
6839                    result: projected,
6840                })),
6841                Err(error) => Poll::Ready(Err(error)),
6842            },
6843            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6844            Poll::Pending => Poll::Pending,
6845        }
6846    }
6847}
6848
6849fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6850    Error::NonDeterministicReplay(ReplayFailure::new(
6851        "recorded_command_mismatch",
6852        Some(recorded.sequence()),
6853        Some(recorded.shape().to_string()),
6854        Some(actual.into()),
6855        "current workflow command does not match the recorded durable command sequence",
6856    ))
6857}
6858
6859pub struct SignalCall {
6860    ctx: WorkflowContext,
6861    signal_name: String,
6862    opened_wait: bool,
6863    matched_pending: bool,
6864}
6865
6866impl SignalCall {
6867    fn poll_avro_value(
6868        mut self: Pin<&mut Self>,
6869        _cx: &mut TaskContext<'_>,
6870    ) -> Poll<Result<Vec<AvroValue>>> {
6871        if self.matched_pending {
6872            return Poll::Pending;
6873        }
6874
6875        let ctx = self.ctx.clone();
6876        let mut state = match ctx.state.lock() {
6877            Ok(state) => state,
6878            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6879        };
6880
6881        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6882            match recorded {
6883                RecordedCommand::SignalWait {
6884                    sequence,
6885                    signal_name,
6886                    value,
6887                } => {
6888                    if signal_name != self.signal_name {
6889                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6890                            ReplayFailure::new(
6891                                "recorded_command_detail_mismatch",
6892                                Some(sequence),
6893                                Some(format!("signal wait:{signal_name}")),
6894                                Some(format!("signal wait:{}", self.signal_name)),
6895                                "recorded signal name differs from the current workflow command",
6896                            ),
6897                        )));
6898                    }
6899
6900                    state.command_cursor += 1;
6901                    if let Some(value) = value {
6902                        return Poll::Ready(Ok(value));
6903                    }
6904                    if state
6905                        .resume_signal
6906                        .as_ref()
6907                        .is_some_and(|signal| signal.signal_name == self.signal_name)
6908                    {
6909                        let signal = state
6910                            .resume_signal
6911                            .take()
6912                            .expect("matching resume signal is present");
6913                        return Poll::Ready(Ok(signal.arguments));
6914                    }
6915
6916                    state.matched_recorded_pending = true;
6917                    self.opened_wait = true;
6918                    self.matched_pending = true;
6919                    return Poll::Pending;
6920                }
6921                other => {
6922                    return Poll::Ready(Err(command_mismatch(
6923                        &other,
6924                        format!("signal wait:{}", self.signal_name),
6925                    )));
6926                }
6927            }
6928        }
6929
6930        if state
6931            .resume_signal
6932            .as_ref()
6933            .is_some_and(|signal| signal.signal_name == self.signal_name)
6934        {
6935            let signal = state
6936                .resume_signal
6937                .take()
6938                .expect("matching resume signal is present");
6939            return Poll::Ready(Ok(signal.arguments));
6940        }
6941
6942        if !self.opened_wait {
6943            state.commands.push(json!({
6944                "type": "open_signal_wait",
6945                "signal_name": self.signal_name
6946            }));
6947            self.opened_wait = true;
6948        }
6949
6950        Poll::Pending
6951    }
6952}
6953
6954impl Future for SignalCall {
6955    type Output = Result<Vec<Value>>;
6956
6957    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6958        match self.poll_avro_value(cx) {
6959            Poll::Ready(Ok(values)) => Poll::Ready(
6960                values
6961                    .into_iter()
6962                    .map(AvroValue::into_json)
6963                    .collect::<Result<Vec<_>>>(),
6964            ),
6965            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6966            Poll::Pending => Poll::Pending,
6967        }
6968    }
6969}
6970
6971#[derive(Clone, Debug)]
6972pub struct ActivityContext {
6973    client: Client,
6974    pub task_id: String,
6975    pub activity_attempt_id: String,
6976    pub lease_owner: String,
6977    pub activity_type: String,
6978    pub attempt_number: u64,
6979    pub task_queue: String,
6980    pub worker_id: String,
6981}
6982
6983impl ActivityContext {
6984    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
6985        self.client
6986            .heartbeat_activity_task(
6987                &self.task_id,
6988                &self.activity_attempt_id,
6989                &self.lease_owner,
6990                details,
6991            )
6992            .await
6993    }
6994}
6995
6996fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
6997    match value {
6998        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
6999            value, codec,
7000        )?)),
7001        None => Ok(AvroValue::Array(Vec::new())),
7002    }
7003}
7004
7005fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7006    let Some(signal_name) = task
7007        .signal_name
7008        .as_deref()
7009        .filter(|value| !value.is_empty())
7010    else {
7011        return Ok(None);
7012    };
7013    let Some(arguments) = task.signal_arguments.as_ref() else {
7014        return Ok(None);
7015    };
7016
7017    let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
7018    let AvroValue::Array(arguments) = decoded else {
7019        unreachable!("normalize_avro_arguments always returns an array");
7020    };
7021
7022    Ok(Some(ResumeSignal {
7023        signal_name: signal_name.to_string(),
7024        arguments,
7025    }))
7026}
7027
7028fn recorded_commands(
7029    events: &[HistoryEvent],
7030    fallback_codec: &str,
7031    parent: WorkflowIdentity,
7032) -> Result<Vec<RecordedCommand>> {
7033    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7034    let mut last_new_sequence = None;
7035
7036    for event in events {
7037        let is_activity = matches!(
7038            event.event_type.as_str(),
7039            "ActivityScheduled"
7040                | "ActivityStarted"
7041                | "ActivityHeartbeatRecorded"
7042                | "ActivityRetryScheduled"
7043                | "ActivityCompleted"
7044                | "ActivityFailed"
7045                | "ActivityCancelled"
7046                | "ActivityTimedOut"
7047        );
7048        let is_workflow_timer = matches!(
7049            event.event_type.as_str(),
7050            "TimerScheduled" | "TimerCancelled" | "TimerFired"
7051        ) && !is_internal_timer_event(event);
7052        let is_child_workflow = matches!(
7053            event.event_type.as_str(),
7054            "ChildWorkflowScheduled"
7055                | "ChildRunCompleted"
7056                | "ChildRunFailed"
7057                | "ChildRunCancelled"
7058                | "ChildRunTerminated"
7059        );
7060        let is_signal_wait = is_recorded_signal_wait_event(event);
7061        let is_side_effect = event.event_type == "SideEffectRecorded";
7062        let is_version_marker = event.event_type == "VersionMarkerRecorded";
7063        if !is_activity
7064            && !is_workflow_timer
7065            && !is_child_workflow
7066            && !is_signal_wait
7067            && !is_side_effect
7068            && !is_version_marker
7069        {
7070            continue;
7071        }
7072
7073        let sequence = durable_event_sequence(event).ok_or_else(|| {
7074            Error::NonDeterministicReplay(ReplayFailure::new(
7075                "durable_command_sequence_missing",
7076                None,
7077                Some("positive workflow sequence".to_string()),
7078                Some(event.event_type.clone()),
7079                "durable command history event has no workflow sequence",
7080            ))
7081        })?;
7082        if sequence == 0 {
7083            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7084                "durable_command_sequence_invalid",
7085                Some(sequence),
7086                Some("positive workflow sequence".to_string()),
7087                Some(sequence.to_string()),
7088                "durable command history uses an invalid workflow sequence",
7089            )));
7090        }
7091        if !events_by_sequence.contains_key(&sequence) {
7092            if let Some(previous) = last_new_sequence {
7093                if sequence < previous {
7094                    return Err(invalid_recorded_history(
7095                        "durable_command_sequence_mismatch",
7096                        sequence,
7097                        &format!("workflow sequence greater than {previous}"),
7098                        &sequence.to_string(),
7099                        "durable commands are not strictly ordered by their recorded workflow sequence",
7100                    ));
7101                }
7102            }
7103            last_new_sequence = Some(sequence);
7104        }
7105        events_by_sequence.entry(sequence).or_default().push(event);
7106    }
7107
7108    let commands: Vec<RecordedCommand> = events_by_sequence
7109        .into_iter()
7110        .map(|(sequence, sequence_events)| {
7111            let activity_events: Vec<_> = sequence_events
7112                .iter()
7113                .copied()
7114                .filter(|event| event.event_type.starts_with("Activity"))
7115                .collect();
7116            let timer_events: Vec<_> = sequence_events
7117                .iter()
7118                .copied()
7119                .filter(|event| event.event_type.starts_with("Timer"))
7120                .collect();
7121            let child_events: Vec<_> = sequence_events
7122                .iter()
7123                .copied()
7124                .filter(|event| {
7125                    event.event_type == "ChildWorkflowScheduled"
7126                        || event.event_type.starts_with("ChildRun")
7127                })
7128                .collect();
7129            let signal_wait_events: Vec<_> = sequence_events
7130                .iter()
7131                .copied()
7132                .filter(|event| is_recorded_signal_wait_event(event))
7133                .collect();
7134            let side_effect_events: Vec<_> = sequence_events
7135                .iter()
7136                .copied()
7137                .filter(|event| event.event_type == "SideEffectRecorded")
7138                .collect();
7139            let version_marker_events: Vec<_> = sequence_events
7140                .iter()
7141                .copied()
7142                .filter(|event| event.event_type == "VersionMarkerRecorded")
7143                .collect();
7144
7145            let command_kind_count = usize::from(!activity_events.is_empty())
7146                + usize::from(!timer_events.is_empty())
7147                + usize::from(!child_events.is_empty())
7148                + usize::from(!signal_wait_events.is_empty())
7149                + usize::from(!side_effect_events.is_empty())
7150                + usize::from(!version_marker_events.is_empty());
7151            if command_kind_count > 1 {
7152                let actual = [
7153                    (!activity_events.is_empty()).then_some("activity"),
7154                    (!timer_events.is_empty()).then_some("timer"),
7155                    (!child_events.is_empty()).then_some("child workflow"),
7156                    (!signal_wait_events.is_empty()).then_some("signal wait"),
7157                    (!side_effect_events.is_empty()).then_some("side effect"),
7158                    (!version_marker_events.is_empty()).then_some("version marker"),
7159                ]
7160                .into_iter()
7161                .flatten()
7162                .collect::<Vec<_>>()
7163                .join(" and ");
7164                return Err(invalid_recorded_history(
7165                    "durable_command_sequence_collision",
7166                    sequence,
7167                    "one durable command kind",
7168                    &actual,
7169                    "one workflow sequence records more than one durable command kind",
7170                ));
7171            }
7172
7173            if !activity_events.is_empty() {
7174                let scheduled_count = activity_events
7175                    .iter()
7176                    .filter(|event| event.event_type == "ActivityScheduled")
7177                    .count();
7178                if scheduled_count > 1 {
7179                    return Err(invalid_recorded_history(
7180                        "duplicate_activity_schedule",
7181                        sequence,
7182                        "at most one ActivityScheduled event",
7183                        "multiple ActivityScheduled events",
7184                        "activity history schedules more than one command at one workflow sequence",
7185                    ));
7186                }
7187                let activity_type = activity_events.iter().find_map(|event| {
7188                    event
7189                        .payload
7190                        .get("activity_type")
7191                        .or_else(|| event.payload.get("activity_name"))
7192                        .and_then(Value::as_str)
7193                        .map(str::to_string)
7194                });
7195                if activity_events.iter().filter_map(|event| {
7196                    event
7197                        .payload
7198                        .get("activity_type")
7199                        .or_else(|| event.payload.get("activity_name"))
7200                        .and_then(Value::as_str)
7201                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7202                    return Err(invalid_recorded_history(
7203                        "activity_identity_mismatch",
7204                        sequence,
7205                        activity_type.as_deref().unwrap_or("one activity identity"),
7206                        "conflicting activity identities",
7207                        "activity lifecycle events at one workflow sequence disagree on identity",
7208                    ));
7209                }
7210                let terminal: Vec<_> = activity_events
7211                    .iter()
7212                    .copied()
7213                    .filter(|event| {
7214                        matches!(
7215                            event.event_type.as_str(),
7216                            "ActivityCompleted"
7217                                | "ActivityFailed"
7218                                | "ActivityCancelled"
7219                                | "ActivityTimedOut"
7220                        )
7221                    })
7222                    .collect();
7223                if terminal.len() > 1 {
7224                    return Err(invalid_recorded_history(
7225                        "duplicate_activity_terminal_event",
7226                        sequence,
7227                        "at most one terminal activity event",
7228                        "multiple terminal activity events",
7229                        "activity history settles one command more than once",
7230                    ));
7231                }
7232                let outcome = terminal
7233                    .first()
7234                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7235                    .transpose()?;
7236                let options = activity_events
7237                    .iter()
7238                    .find(|event| event.event_type == "ActivityScheduled")
7239                    .and_then(|event| event.payload.get("activity"))
7240                    .and_then(Value::as_object)
7241                    .map(|activity| RecordedActivityOptions {
7242                        task_queue: recorded_optional_string(activity, "queue"),
7243                        execution_mode: recorded_optional_string(activity, "execution_mode"),
7244                        retry_policy: recorded_activity_retry_snapshot(
7245                            activity.get("retry_policy"),
7246                        ),
7247                    });
7248                return Ok(RecordedCommand::Activity {
7249                    sequence,
7250                    activity_type,
7251                    options,
7252                    outcome,
7253                });
7254            }
7255
7256            if !child_events.is_empty() {
7257                let scheduled: Vec<_> = child_events
7258                    .iter()
7259                    .copied()
7260                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
7261                    .collect();
7262                if scheduled.len() != 1 {
7263                    return Err(invalid_recorded_history(
7264                        "child_workflow_schedule_missing_or_duplicate",
7265                        sequence,
7266                        "one ChildWorkflowScheduled event",
7267                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7268                        "child workflow replay requires exactly one recorded schedule event",
7269                    ));
7270                }
7271                let workflow_type = child_events.iter().find_map(|event| {
7272                    event
7273                        .payload
7274                        .get("child_workflow_type")
7275                        .or_else(|| event.payload.get("workflow_type"))
7276                        .and_then(Value::as_str)
7277                        .filter(|value| !value.is_empty())
7278                        .map(str::to_string)
7279                });
7280                if child_events
7281                    .iter()
7282                    .filter_map(|event| {
7283                        event
7284                            .payload
7285                            .get("child_workflow_type")
7286                            .or_else(|| event.payload.get("workflow_type"))
7287                            .and_then(Value::as_str)
7288                    })
7289                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
7290                {
7291                    return Err(invalid_recorded_history(
7292                        "child_workflow_identity_mismatch",
7293                        sequence,
7294                        workflow_type
7295                            .as_deref()
7296                            .unwrap_or("one child workflow type"),
7297                        "conflicting child workflow types",
7298                        "child workflow lifecycle events at one sequence disagree on type",
7299                    ));
7300                }
7301                let mut outcomes = child_workflow_outcomes(
7302                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7303                    fallback_codec,
7304                    parent.clone(),
7305                )?;
7306                if outcomes.len() > 1 {
7307                    return Err(invalid_recorded_history(
7308                        "duplicate_child_workflow_terminal_event",
7309                        sequence,
7310                        "at most one terminal child event",
7311                        "multiple terminal child events",
7312                        "child workflow history settles one command more than once",
7313                    ));
7314                }
7315                return Ok(RecordedCommand::ChildWorkflow {
7316                    sequence,
7317                    workflow_type,
7318                    outcome: outcomes.pop(),
7319                });
7320            }
7321
7322            if !signal_wait_events.is_empty() {
7323                let opened: Vec<_> = signal_wait_events
7324                    .iter()
7325                    .copied()
7326                    .filter(|event| event.event_type == "SignalWaitOpened")
7327                    .collect();
7328                if opened.len() != 1 {
7329                    return Err(invalid_recorded_history(
7330                        "signal_wait_open_missing_or_duplicate",
7331                        sequence,
7332                        "one SignalWaitOpened event",
7333                        &format!("{} SignalWaitOpened events", opened.len()),
7334                        "signal replay requires exactly one canonical wait-open event",
7335                    ));
7336                }
7337
7338                let applied: Vec<_> = signal_wait_events
7339                    .iter()
7340                    .copied()
7341                    .filter(|event| event.event_type == "SignalApplied")
7342                    .collect();
7343                if applied.len() > 1 {
7344                    return Err(invalid_recorded_history(
7345                        "duplicate_signal_wait_apply",
7346                        sequence,
7347                        "at most one SignalApplied event",
7348                        "multiple SignalApplied events",
7349                        "signal history applies one durable wait more than once",
7350                    ));
7351                }
7352
7353                let signal_names = signal_wait_events
7354                    .iter()
7355                    .map(|event| required_signal_wait_name(event, sequence))
7356                    .collect::<Result<Vec<_>>>()?;
7357                let signal_name = signal_names
7358                    .first()
7359                    .expect("signal wait events are not empty")
7360                    .clone();
7361                if signal_names.iter().any(|candidate| candidate != &signal_name) {
7362                    return Err(invalid_recorded_history(
7363                        "signal_wait_identity_mismatch",
7364                        sequence,
7365                        &signal_name,
7366                        "conflicting signal names",
7367                        "signal wait lifecycle events at one workflow sequence disagree on identity",
7368                    ));
7369                }
7370                let value = applied
7371                    .first()
7372                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
7373                    .transpose()?;
7374                return Ok(RecordedCommand::SignalWait {
7375                    sequence,
7376                    signal_name,
7377                    value,
7378                });
7379            }
7380
7381            if !side_effect_events.is_empty() {
7382                if side_effect_events.len() != 1 {
7383                    return Err(invalid_recorded_history(
7384                        "duplicate_side_effect_record",
7385                        sequence,
7386                        "one SideEffectRecorded event",
7387                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
7388                        "side-effect history records one workflow command more than once",
7389                    ));
7390                }
7391                let event = side_effect_events[0];
7392                let result = event.payload.get("result").ok_or_else(|| {
7393                    invalid_recorded_history(
7394                        "side_effect_result_missing",
7395                        sequence,
7396                        "recorded result payload",
7397                        "missing result",
7398                        "side-effect history is missing its recorded value",
7399                    )
7400                })?;
7401                let has_published_envelope = result.as_str().is_some()
7402                    || result.as_object().is_some_and(|envelope| {
7403                        envelope.get("codec").and_then(Value::as_str).is_some()
7404                            && envelope.get("blob").and_then(Value::as_str).is_some()
7405                    });
7406                if !has_published_envelope {
7407                    return Err(invalid_recorded_history(
7408                        "side_effect_payload_malformed",
7409                        sequence,
7410                        "payload blob or {codec, blob} envelope",
7411                        &result.to_string(),
7412                        "side-effect history result does not use a published payload envelope",
7413                    ));
7414                }
7415                let codec = event
7416                    .payload
7417                    .get("payload_codec")
7418                    .and_then(Value::as_str)
7419                    .unwrap_or(fallback_codec);
7420                let value = decode_wire_avro_value(result, codec).map_err(|error| {
7421                    invalid_recorded_history(
7422                        "side_effect_payload_incompatible",
7423                        sequence,
7424                        &format!("valid {codec} payload envelope"),
7425                        &error.to_string(),
7426                        "side-effect history payload cannot be decoded with its recorded codec",
7427                    )
7428                })?;
7429                return Ok(RecordedCommand::SideEffect { sequence, value });
7430            }
7431
7432            if !version_marker_events.is_empty() {
7433                if version_marker_events.len() != 1 {
7434                    return Err(invalid_recorded_history(
7435                        "duplicate_version_marker_record",
7436                        sequence,
7437                        "one VersionMarkerRecorded event",
7438                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7439                        "version-marker history records one workflow command more than once",
7440                    ));
7441                }
7442                let payload = &version_marker_events[0].payload;
7443                let change_id = payload
7444                    .get("change_id")
7445                    .and_then(Value::as_str)
7446                    .filter(|value| !value.is_empty())
7447                    .map(str::to_string)
7448                    .ok_or_else(|| {
7449                        invalid_recorded_history(
7450                            "version_marker_field_missing",
7451                            sequence,
7452                            "non-empty change_id",
7453                            "missing or invalid change_id",
7454                            "version-marker history is missing its stable change ID",
7455                        )
7456                    })?;
7457                let version = required_version_i32(payload, "version", sequence)?;
7458                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7459                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7460                if min_supported > max_supported || version < min_supported || version > max_supported {
7461                    return Err(invalid_recorded_history(
7462                        "version_marker_history_range_invalid",
7463                        sequence,
7464                        "min_supported <= version <= max_supported",
7465                        &format!("{min_supported} <= {version} <= {max_supported}"),
7466                        "recorded version marker contains an internally incompatible range",
7467                    ));
7468                }
7469                return Ok(RecordedCommand::VersionMarker {
7470                    sequence,
7471                    change_id,
7472                    version,
7473                });
7474            }
7475
7476            let scheduled: Vec<_> = timer_events
7477                .iter()
7478                .copied()
7479                .filter(|event| event.event_type == "TimerScheduled")
7480                .collect();
7481            let fired: Vec<_> = timer_events
7482                .iter()
7483                .copied()
7484                .filter(|event| event.event_type == "TimerFired")
7485                .collect();
7486            if scheduled.len() != 1 {
7487                return Err(invalid_recorded_history(
7488                    "timer_schedule_missing_or_duplicate",
7489                    sequence,
7490                    "one TimerScheduled event",
7491                    &format!("{} TimerScheduled events", scheduled.len()),
7492                    "timer replay requires exactly one recorded schedule event",
7493                ));
7494            }
7495            if fired.len() > 1 {
7496                return Err(invalid_recorded_history(
7497                    "duplicate_timer_fire",
7498                    sequence,
7499                    "at most one TimerFired event",
7500                    "multiple TimerFired events",
7501                    "timer history contains more than one fire event for a workflow sequence",
7502                ));
7503            }
7504
7505            let scheduled = scheduled[0];
7506            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7507            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7508            if let Some(fired) = fired.first() {
7509                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7510                if fired_timer_id != timer_id {
7511                    return Err(invalid_recorded_history(
7512                        "timer_identity_mismatch",
7513                        sequence,
7514                        &timer_id,
7515                        &fired_timer_id,
7516                        "TimerFired does not correspond to the recorded TimerScheduled event",
7517                    ));
7518                }
7519                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7520                if fired_delay != delay_seconds {
7521                    return Err(invalid_recorded_history(
7522                        "timer_history_delay_mismatch",
7523                        sequence,
7524                        &delay_seconds.to_string(),
7525                        &fired_delay.to_string(),
7526                        "TimerScheduled and TimerFired record different delays",
7527                    ));
7528                }
7529            }
7530
7531            Ok(RecordedCommand::Timer {
7532                sequence,
7533                delay_seconds,
7534                fired: !fired.is_empty(),
7535            })
7536        })
7537        .collect::<Result<_>>()?;
7538
7539    let mut marker_sequences = HashMap::new();
7540    for command in &commands {
7541        if let RecordedCommand::VersionMarker {
7542            sequence,
7543            change_id,
7544            ..
7545        } = command
7546        {
7547            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7548                return Err(invalid_recorded_history(
7549                    "duplicate_version_marker",
7550                    *sequence,
7551                    &format!("one marker for change ID {change_id:?}"),
7552                    &format!("markers at sequences {first_sequence} and {sequence}"),
7553                    "workflow history contains duplicate markers for one stable change ID",
7554                ));
7555            }
7556        }
7557    }
7558
7559    Ok(commands)
7560}
7561
7562fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7563    payload
7564        .get(field)
7565        .and_then(Value::as_i64)
7566        .and_then(|value| i32::try_from(value).ok())
7567        .ok_or_else(|| {
7568            invalid_recorded_history(
7569                "version_marker_field_missing",
7570                sequence,
7571                &format!("integer {field}"),
7572                "missing or out-of-range integer",
7573                "version-marker history is missing a required integer field",
7574            )
7575        })
7576}
7577
7578fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7579    event
7580        .payload
7581        .get("sequence")
7582        .or_else(|| event.payload.get("workflow_sequence"))
7583        .or_else(|| event.raw.get("sequence"))
7584        .or_else(|| event.raw.get("workflow_sequence"))
7585        .and_then(value_as_u64)
7586}
7587
7588fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7589    matches!(
7590        event
7591            .payload
7592            .get("timer_kind")
7593            .or_else(|| event.raw.get("timer_kind"))
7594            .and_then(Value::as_str),
7595        Some("condition_timeout" | "signal_timeout")
7596    )
7597}
7598
7599fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7600    event
7601        .payload
7602        .get("signal_name")
7603        .or_else(|| event.raw.get("signal_name"))
7604        .and_then(Value::as_str)
7605        .filter(|value| !value.is_empty())
7606        .map(str::to_string)
7607        .ok_or_else(|| {
7608            invalid_recorded_history(
7609                "signal_wait_name_missing",
7610                sequence,
7611                "non-empty signal_name",
7612                &event.event_type,
7613                "canonical signal-wait history is missing its signal identity",
7614            )
7615        })
7616}
7617
7618fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7619    matches!(
7620        event.event_type.as_str(),
7621        "SignalWaitOpened" | "SignalApplied"
7622    )
7623}
7624
7625fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7626    event
7627        .payload
7628        .get(field)
7629        .and_then(Value::as_str)
7630        .filter(|value| !value.is_empty())
7631        .map(str::to_string)
7632        .ok_or_else(|| {
7633            invalid_recorded_history(
7634                "timer_history_field_missing",
7635                sequence,
7636                field,
7637                &event.event_type,
7638                "timer history is missing a required identity field",
7639            )
7640        })
7641}
7642
7643fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7644    event
7645        .payload
7646        .get(field)
7647        .and_then(value_as_u64)
7648        .ok_or_else(|| {
7649            invalid_recorded_history(
7650                "timer_history_field_missing",
7651                sequence,
7652                field,
7653                &event.event_type,
7654                "timer history is missing a required numeric field",
7655            )
7656        })
7657}
7658
7659fn invalid_recorded_history(
7660    reason: &str,
7661    sequence: u64,
7662    expected: &str,
7663    actual: &str,
7664    message: &str,
7665) -> Error {
7666    Error::NonDeterministicReplay(ReplayFailure::new(
7667        reason,
7668        Some(sequence),
7669        Some(expected.to_string()),
7670        Some(actual.to_string()),
7671        message,
7672    ))
7673}
7674
7675type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7676
7677fn activity_outcome(
7678    event: &HistoryEvent,
7679    fallback_codec: &str,
7680    recorded_activity_type: Option<String>,
7681) -> Result<ActivityOutcome> {
7682    if event.event_type == "ActivityCompleted" {
7683        let codec = event
7684            .payload
7685            .get("payload_codec")
7686            .and_then(Value::as_str)
7687            .unwrap_or(fallback_codec);
7688        return Ok(Ok(decode_wire_avro_value(
7689            event.payload.get("result").unwrap_or(&Value::Null),
7690            codec,
7691        )?));
7692    }
7693
7694    let payload = &event.payload;
7695    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7696        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7697        "ActivityCancelled" => (
7698            ActivityFailureKind::Cancelled,
7699            "cancelled",
7700            "activity was cancelled",
7701        ),
7702        "ActivityTimedOut" => (
7703            ActivityFailureKind::TimedOut,
7704            "timeout",
7705            "activity timed out",
7706        ),
7707        _ => unreachable!("activity_outcome is called only for terminal activity events"),
7708    };
7709    let exception = payload
7710        .get("exception")
7711        .filter(|value| !value.is_null())
7712        .cloned();
7713    let failure_category = payload_string(payload, "failure_category");
7714    let timeout_kind = payload_string(payload, "timeout_kind");
7715    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7716        ActivityFailureKind::Failed => failure_category
7717            .clone()
7718            .unwrap_or_else(|| fallback_reason.to_string()),
7719        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7720        ActivityFailureKind::TimedOut => timeout_kind
7721            .clone()
7722            .unwrap_or_else(|| fallback_reason.to_string()),
7723    });
7724    let message = payload_string(payload, "message")
7725        .or_else(|| {
7726            exception
7727                .as_ref()
7728                .and_then(|value| payload_string(value, "message"))
7729        })
7730        .unwrap_or_else(|| fallback_message.to_string());
7731
7732    Ok(Err(ActivityFailure {
7733        kind,
7734        reason,
7735        message,
7736        activity_execution_id: payload_string(payload, "activity_execution_id"),
7737        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7738        activity_type: payload_string(payload, "activity_type")
7739            .or_else(|| payload_string(payload, "activity_name"))
7740            .or(recorded_activity_type),
7741        activity_class: payload_string(payload, "activity_class"),
7742        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7743        failure_id: payload_string(payload, "failure_id"),
7744        failure_category,
7745        timeout_kind,
7746        non_retryable: payload
7747            .get("non_retryable")
7748            .and_then(Value::as_bool)
7749            .unwrap_or(false),
7750        exception_type: payload_string(payload, "exception_type").or_else(|| {
7751            exception
7752                .as_ref()
7753                .and_then(|value| payload_string(value, "type"))
7754        }),
7755        exception_class: payload_string(payload, "exception_class").or_else(|| {
7756            exception
7757                .as_ref()
7758                .and_then(|value| payload_string(value, "class"))
7759        }),
7760        code: payload
7761            .get("code")
7762            .filter(|value| !value.is_null())
7763            .cloned(),
7764        exception,
7765    }))
7766}
7767
7768type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7769
7770fn child_workflow_outcomes(
7771    events: &[HistoryEvent],
7772    fallback_codec: &str,
7773    parent: WorkflowIdentity,
7774) -> Result<Vec<ChildWorkflowOutcome>> {
7775    let mut outcomes = Vec::new();
7776
7777    for event in events {
7778        let kind = match event.event_type.as_str() {
7779            "ChildRunCompleted" => None,
7780            "ChildRunFailed" => Some((
7781                ChildWorkflowFailureKind::Failed,
7782                "child_workflow",
7783                "child workflow failed",
7784            )),
7785            "ChildRunCancelled" => Some((
7786                ChildWorkflowFailureKind::Cancelled,
7787                "cancelled",
7788                "child workflow was cancelled",
7789            )),
7790            "ChildRunTerminated" => Some((
7791                ChildWorkflowFailureKind::Terminated,
7792                "terminated",
7793                "child workflow was terminated",
7794            )),
7795            _ => continue,
7796        };
7797        let payload = &event.payload;
7798        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7799        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7800        let child_workflow_type = payload_string(payload, "child_workflow_type");
7801
7802        if let Some((kind, reason, fallback_message)) = kind {
7803            let exception = payload
7804                .get("exception")
7805                .filter(|value| !value.is_null())
7806                .cloned();
7807            let message = payload_string(payload, "message")
7808                .or_else(|| {
7809                    exception
7810                        .as_ref()
7811                        .and_then(|value| payload_string(value, "message"))
7812                })
7813                .unwrap_or_else(|| fallback_message.to_string());
7814            let exception_type = payload_string(payload, "exception_type").or_else(|| {
7815                exception
7816                    .as_ref()
7817                    .and_then(|value| payload_string(value, "type"))
7818            });
7819            let exception_class = payload_string(payload, "exception_class").or_else(|| {
7820                exception
7821                    .as_ref()
7822                    .and_then(|value| payload_string(value, "class"))
7823            });
7824            outcomes.push(Err(ChildWorkflowFailure {
7825                kind,
7826                reason: reason.to_string(),
7827                message,
7828                parent_workflow_id: parent.workflow_id.clone(),
7829                parent_workflow_run_id: parent.run_id.clone(),
7830                child_workflow_id,
7831                child_workflow_run_id,
7832                child_workflow_type,
7833                failure_id: payload_string(payload, "failure_id"),
7834                failure_category: payload_string(payload, "failure_category"),
7835                exception_type,
7836                exception_class,
7837                non_retryable: payload
7838                    .get("non_retryable")
7839                    .and_then(Value::as_bool)
7840                    .unwrap_or(false),
7841                code: payload
7842                    .get("code")
7843                    .filter(|value| !value.is_null())
7844                    .cloned(),
7845                exception,
7846            }));
7847            continue;
7848        }
7849
7850        let codec = payload
7851            .get("payload_codec")
7852            .and_then(Value::as_str)
7853            .unwrap_or(fallback_codec);
7854        let result = payload
7855            .get("result")
7856            .or_else(|| payload.get("output"))
7857            .unwrap_or(&Value::Null);
7858        outcomes.push(Ok(ChildWorkflowAvroResult {
7859            parent: parent.clone(),
7860            child: WorkflowIdentity {
7861                workflow_id: child_workflow_id,
7862                run_id: child_workflow_run_id,
7863            },
7864            child_workflow_type,
7865            result: decode_wire_avro_value(result, codec)?,
7866        }));
7867    }
7868
7869    Ok(outcomes)
7870}
7871
7872fn payload_string(payload: &Value, key: &str) -> Option<String> {
7873    payload
7874        .get(key)
7875        .and_then(Value::as_str)
7876        .filter(|value| !value.is_empty())
7877        .map(str::to_string)
7878}
7879
7880fn workflow_failure_command(error: &Error) -> Value {
7881    let (exception_type, exception_class, properties) = match error {
7882        Error::ActivityFailed(failure) => (
7883            match failure.kind {
7884                ActivityFailureKind::Failed => "ActivityFailed",
7885                ActivityFailureKind::Cancelled => "ActivityCancelled",
7886                ActivityFailureKind::TimedOut => "ActivityTimedOut",
7887            },
7888            "durable_workflow::ActivityFailure",
7889            json!({
7890                "reason": failure.reason,
7891                "activity_execution_id": failure.activity_execution_id,
7892                "activity_attempt_id": failure.activity_attempt_id,
7893                "activity_type": failure.activity_type,
7894                "activity_class": failure.activity_class,
7895                "attempt_number": failure.attempt_number,
7896                "failure_id": failure.failure_id,
7897                "failure_category": failure.failure_category,
7898                "timeout_kind": failure.timeout_kind,
7899                "activity_non_retryable": failure.non_retryable,
7900                "activity_exception_type": failure.exception_type,
7901                "activity_exception_class": failure.exception_class,
7902                "activity_code": failure.code,
7903                "activity_exception": failure.exception,
7904            }),
7905        ),
7906        Error::ChildWorkflowFailed(failure) => (
7907            match failure.kind {
7908                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7909                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7910                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7911            },
7912            "durable_workflow::ChildWorkflowFailure",
7913            json!({
7914                "reason": failure.reason,
7915                "parent_workflow_id": failure.parent_workflow_id,
7916                "parent_workflow_run_id": failure.parent_workflow_run_id,
7917                "child_workflow_id": failure.child_workflow_id,
7918                "child_workflow_run_id": failure.child_workflow_run_id,
7919                "child_workflow_type": failure.child_workflow_type,
7920                "failure_id": failure.failure_id,
7921                "failure_category": failure.failure_category,
7922                "child_exception_type": failure.exception_type,
7923                "child_exception_class": failure.exception_class,
7924                "child_non_retryable": failure.non_retryable,
7925                "child_code": failure.code,
7926                "child_exception": failure.exception,
7927            }),
7928        ),
7929        Error::NonDeterministicReplay(_) => (
7930            "NonDeterministicReplay",
7931            "durable_workflow::Error",
7932            Value::Null,
7933        ),
7934        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7935    };
7936    let non_retryable = match error {
7937        Error::ActivityFailed(failure) => failure.non_retryable,
7938        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7939        Error::NonDeterministicReplay(_) => true,
7940        _ => false,
7941    };
7942
7943    json!({
7944        "type": "fail_workflow",
7945        "message": error.to_string(),
7946        "exception_type": exception_type,
7947        "exception_class": exception_class,
7948        "non_retryable": non_retryable,
7949        "exception": {
7950            "type": exception_type,
7951            "class": exception_class,
7952            "message": error.to_string(),
7953            "properties": properties,
7954        }
7955    })
7956}
7957
7958fn workflow_task_integrity_error(error: &Error) -> bool {
7959    matches!(
7960        error,
7961        Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7962    )
7963}
7964
7965fn decode_signal_event_arguments(
7966    event: &HistoryEvent,
7967    fallback_codec: &str,
7968) -> Result<Vec<AvroValue>> {
7969    let codec = event
7970        .payload
7971        .get("payload_codec")
7972        .and_then(Value::as_str)
7973        .unwrap_or(fallback_codec);
7974    let raw = event
7975        .payload
7976        .get("value")
7977        .or_else(|| event.payload.get("input"))
7978        .or_else(|| event.payload.get("arguments"));
7979    let decoded = match raw.filter(|value| !value.is_null()) {
7980        Some(value) => decode_wire_avro_value(value, codec)?,
7981        None => AvroValue::Array(Vec::new()),
7982    };
7983    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
7984        unreachable!("normalize_avro_arguments always returns an array");
7985    };
7986    Ok(arguments)
7987}
7988
7989fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7990    let Some(export_events) = task
7991        .history_export
7992        .as_ref()
7993        .and_then(|export| export.get("history_events"))
7994        .and_then(Value::as_array)
7995    else {
7996        return Ok(());
7997    };
7998
7999    if export_events.len() > task.history_events.len() {
8000        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8001    }
8002
8003    Ok(())
8004}
8005
8006fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8007    let Some(export) = task.history_export.as_ref() else {
8008        return Ok(());
8009    };
8010    let signals = export
8011        .get("signals")
8012        .and_then(Value::as_array)
8013        .cloned()
8014        .unwrap_or_default();
8015    let activities = export
8016        .get("activities")
8017        .and_then(Value::as_array)
8018        .cloned()
8019        .unwrap_or_default();
8020    let export_codec = export
8021        .get("payloads")
8022        .and_then(|payloads| payloads.get("codec"))
8023        .and_then(Value::as_str)
8024        .unwrap_or(&task.payload_codec)
8025        .to_string();
8026    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8027
8028    for event in &mut task.history_events {
8029        if event.event_type == "ActivityCompleted" {
8030            let sequence = event
8031                .payload
8032                .get("sequence")
8033                .or_else(|| event.payload.get("workflow_sequence"))
8034                .and_then(value_as_u64);
8035            let Some(activity) = sequence.and_then(|sequence| {
8036                activities.iter().find(|activity| {
8037                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8038                })
8039            }) else {
8040                continue;
8041            };
8042            let Some(payload) = event.payload.as_object_mut() else {
8043                continue;
8044            };
8045            if missing_payload(payload.get("result")) {
8046                if let Some(result) = activity
8047                    .get("result")
8048                    .filter(|value| !missing_payload(Some(value)))
8049                {
8050                    payload.insert("result".to_string(), result.clone());
8051                }
8052            }
8053            for field in ["payload_codec", "activity_type"] {
8054                if payload
8055                    .get(field)
8056                    .and_then(Value::as_str)
8057                    .unwrap_or_default()
8058                    .is_empty()
8059                {
8060                    if let Some(value) = activity.get(field) {
8061                        payload.insert(field.to_string(), value.clone());
8062                    }
8063                }
8064            }
8065            continue;
8066        }
8067
8068        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8069            continue;
8070        }
8071        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8072        let command_id = event
8073            .payload
8074            .get("workflow_command_id")
8075            .or_else(|| event.raw.get("workflow_command_id"))
8076            .and_then(Value::as_str);
8077        let signal_name = event
8078            .payload
8079            .get("signal_name")
8080            .and_then(Value::as_str)
8081            .unwrap_or_default()
8082            .to_string();
8083        let matched = signals
8084            .iter()
8085            .find(|signal| {
8086                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8087            })
8088            .or_else(|| {
8089                signals.iter().find(|signal| {
8090                    command_id.is_some()
8091                        && signal.get("command_id").and_then(Value::as_str) == command_id
8092                })
8093            })
8094            .or_else(|| {
8095                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8096                let signal = signals
8097                    .iter()
8098                    .filter(|signal| {
8099                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8100                    })
8101                    .nth(*offset);
8102                if signal.is_some() {
8103                    *offset += 1;
8104                }
8105                signal
8106            });
8107        let Some(signal) = matched else {
8108            continue;
8109        };
8110        let signal_codec = signal
8111            .get("payload_codec")
8112            .and_then(Value::as_str)
8113            .unwrap_or(&export_codec);
8114        let Some(payload) = event.payload.as_object_mut() else {
8115            continue;
8116        };
8117        if missing_payload(payload.get("arguments")) {
8118            if let Some(arguments) = signal
8119                .get("arguments")
8120                .filter(|value| !missing_payload(Some(value)))
8121            {
8122                let envelope = match arguments {
8123                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8124                    other => other.clone(),
8125                };
8126                payload.insert("arguments".to_string(), envelope);
8127            }
8128        }
8129        if payload
8130            .get("payload_codec")
8131            .and_then(Value::as_str)
8132            .unwrap_or_default()
8133            .is_empty()
8134        {
8135            payload.insert("payload_codec".to_string(), json!(signal_codec));
8136        }
8137    }
8138
8139    Ok(())
8140}
8141
8142fn missing_payload(value: Option<&Value>) -> bool {
8143    match value {
8144        None | Some(Value::Null) => true,
8145        Some(Value::String(value)) => value.is_empty(),
8146        Some(_) => false,
8147    }
8148}
8149
8150fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8151    let export_signals = task
8152        .history_export
8153        .as_ref()
8154        .and_then(|export| export.get("signals"))
8155        .and_then(Value::as_array)
8156        .cloned()
8157        .unwrap_or_default();
8158    let export_codec = task
8159        .history_export
8160        .as_ref()
8161        .and_then(|export| export.get("payloads"))
8162        .and_then(|payloads| payloads.get("codec"))
8163        .and_then(Value::as_str)
8164        .unwrap_or(&task.payload_codec);
8165    let mut name_offsets: HashMap<String, usize> = HashMap::new();
8166    let mut signals = Vec::new();
8167
8168    for event in &task.history_events {
8169        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8170            continue;
8171        }
8172
8173        let name = event
8174            .payload
8175            .get("signal_name")
8176            .and_then(Value::as_str)
8177            .unwrap_or_default();
8178        if name.is_empty() {
8179            continue;
8180        }
8181        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8182        let command_id = event
8183            .payload
8184            .get("workflow_command_id")
8185            .or_else(|| event.raw.get("workflow_command_id"))
8186            .and_then(Value::as_str);
8187        let matched_export = export_signals
8188            .iter()
8189            .find(|candidate| {
8190                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8191            })
8192            .or_else(|| {
8193                export_signals.iter().find(|candidate| {
8194                    command_id.is_some()
8195                        && candidate.get("command_id").and_then(Value::as_str) == command_id
8196                })
8197            })
8198            .or_else(|| {
8199                let offset = name_offsets.entry(name.to_string()).or_default();
8200                let candidate = export_signals
8201                    .iter()
8202                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8203                    .nth(*offset);
8204                if candidate.is_some() {
8205                    *offset += 1;
8206                }
8207                candidate
8208            });
8209        let codec = event
8210            .payload
8211            .get("payload_codec")
8212            .and_then(Value::as_str)
8213            .or_else(|| {
8214                matched_export
8215                    .and_then(|signal| signal.get("payload_codec"))
8216                    .and_then(Value::as_str)
8217            })
8218            .unwrap_or(export_codec);
8219        let raw_arguments = event
8220            .payload
8221            .get("value")
8222            .or_else(|| event.payload.get("input"))
8223            .or_else(|| event.payload.get("arguments"))
8224            .filter(|value| !value.is_null())
8225            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8226        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8227        let workflow_sequence = event
8228            .payload
8229            .get("workflow_sequence")
8230            .and_then(value_as_u64)
8231            .or_else(|| {
8232                matched_export
8233                    .and_then(|signal| signal.get("workflow_sequence"))
8234                    .and_then(value_as_u64)
8235            });
8236
8237        signals.push(QuerySignal {
8238            id: signal_id.map(str::to_string).or_else(|| {
8239                matched_export
8240                    .and_then(|signal| signal.get("id"))
8241                    .and_then(Value::as_str)
8242                    .map(str::to_string)
8243            }),
8244            name: name.to_string(),
8245            arguments,
8246            avro_arguments,
8247            workflow_sequence,
8248        });
8249    }
8250
8251    if signals.is_empty() {
8252        for signal in export_signals {
8253            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8254                continue;
8255            }
8256            let Some(name) = signal.get("name").and_then(Value::as_str) else {
8257                continue;
8258            };
8259            let codec = signal
8260                .get("payload_codec")
8261                .and_then(Value::as_str)
8262                .unwrap_or(export_codec);
8263            let (arguments, avro_arguments) =
8264                decode_query_signal_arguments(signal.get("arguments"), codec)?;
8265            signals.push(QuerySignal {
8266                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8267                name: name.to_string(),
8268                arguments,
8269                avro_arguments,
8270                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8271            });
8272        }
8273        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8274    }
8275
8276    Ok(signals)
8277}
8278
8279fn decode_query_signal_arguments(
8280    raw: Option<&Value>,
8281    codec: &str,
8282) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8283    let decoded = match raw.filter(|value| !value.is_null()) {
8284        Some(value) => decode_wire_avro_value(value, codec)?,
8285        None => AvroValue::Array(Vec::new()),
8286    };
8287    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8288        unreachable!("normalize_avro_arguments always returns an array");
8289    };
8290    let arguments = avro_arguments
8291        .iter()
8292        .cloned()
8293        .map(AvroValue::into_json)
8294        .collect::<Result<Vec<_>>>()?;
8295    Ok((arguments, avro_arguments))
8296}
8297
8298fn value_as_u64(value: &Value) -> Option<u64> {
8299    value
8300        .as_u64()
8301        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8302}
8303
8304#[cfg(test)]
8305mod tests {
8306    use super::*;
8307    use std::{
8308        io::{Read, Write},
8309        net::{SocketAddr, TcpListener, TcpStream},
8310        sync::atomic::AtomicUsize,
8311        thread,
8312    };
8313
8314    #[test]
8315    fn client_builder_rejects_the_sdk_owned_api_suffix() {
8316        for base_url in [
8317            "http://127.0.0.1:8080/api",
8318            "http://localhost:8080/api/",
8319            "https://runtime.example.test/namespaces/orders/api",
8320        ] {
8321            let error = Client::builder(base_url)
8322                .build()
8323                .expect_err("SDK-owned /api suffix must be rejected during build");
8324
8325            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8326            assert!(
8327                error.to_string().contains("SDK appends /api automatically"),
8328                "the validation error must explain how to fix the endpoint"
8329            );
8330        }
8331    }
8332
8333    #[test]
8334    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8335        for (base_url, expected) in [
8336            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8337            (
8338                "http://localhost:8080/durable-workflow/",
8339                "http://localhost:8080/durable-workflow",
8340            ),
8341            (
8342                "https://runtime.example.test/namespaces/orders",
8343                "https://runtime.example.test/namespaces/orders",
8344            ),
8345            (
8346                "https://runtime.example.test/gateway/api/namespaces/orders",
8347                "https://runtime.example.test/gateway/api/namespaces/orders",
8348            ),
8349            (
8350                "https://api.example.test/runtime/orders/",
8351                "https://api.example.test/runtime/orders",
8352            ),
8353        ] {
8354            let client = Client::builder(base_url)
8355                .build()
8356                .expect("Server and Cloud runtime base URL must remain valid");
8357
8358            assert_eq!(client.base_url, expected);
8359        }
8360    }
8361
8362    fn typed_fidelity_probe() -> AvroValue {
8363        AvroValue::Map(BTreeMap::from([
8364            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8365            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8366            (
8367                "numeric".to_string(),
8368                AvroValue::Map(BTreeMap::from([
8369                    ("0".to_string(), AvroValue::String("zero".to_string())),
8370                    ("1".to_string(), AvroValue::String("one".to_string())),
8371                ])),
8372            ),
8373            (
8374                "nested".to_string(),
8375                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8376                    "enabled".to_string(),
8377                    AvroValue::Boolean(true),
8378                )]))]),
8379            ),
8380            (
8381                "projection_collisions".to_string(),
8382                AvroValue::Array(projection_collision_probe()),
8383            ),
8384        ]))
8385    }
8386
8387    fn projection_collision_probe() -> Vec<AvroValue> {
8388        vec![
8389            AvroValue::Map(BTreeMap::from([
8390                ("$type".to_string(), AvroValue::String("bytes".to_string())),
8391                (
8392                    "base64".to_string(),
8393                    AvroValue::String("ordinary user text".to_string()),
8394                ),
8395            ])),
8396            AvroValue::Map(BTreeMap::from([
8397                ("$type".to_string(), AvroValue::String("map".to_string())),
8398                (
8399                    "entries".to_string(),
8400                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8401                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
8402                        (
8403                            "value".to_string(),
8404                            AvroValue::String("user map".to_string()),
8405                        ),
8406                    ]))]),
8407                ),
8408            ])),
8409        ]
8410    }
8411
8412    #[derive(Clone, Debug, Default, PartialEq)]
8413    struct ReplayCounterState {
8414        loaded: Option<String>,
8415        count: i64,
8416        finished: bool,
8417    }
8418
8419    fn replay_counter_worker() -> Worker {
8420        let client = Client::new("http://127.0.0.1:8080").expect("client");
8421        let mut worker = Worker::new(client, "rust-workers");
8422        worker.register_replayed_workflow(
8423            "replay-counter",
8424            ReplayCounterState::default,
8425            |ctx, _input, state| async move {
8426                let loaded = ctx.activity("load-counter", json!([])).await?;
8427                state.update(|current| {
8428                    current.loaded = loaded.as_str().map(str::to_string);
8429                })?;
8430                for _ in 0..2 {
8431                    let signal = ctx.wait_signal("increment").await?;
8432                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8433                    state.update(|current| current.count += amount)?;
8434                }
8435                state.update(|current| current.finished = true)?;
8436                state.read(|current| Ok(json!(current.count)))?
8437            },
8438        );
8439        worker.register_replayed_query::<ReplayCounterState, _, _>(
8440            "replay-counter",
8441            "current",
8442            |_ctx, state, _args| async move {
8443                Ok(json!({
8444                    "loaded": state.loaded,
8445                    "count": state.count,
8446                    "finished": state.finished,
8447                }))
8448            },
8449        );
8450        worker.register_replayed_query::<ReplayCounterState, _, _>(
8451            "replay-counter",
8452            "detached-mutation",
8453            |_ctx, state, _args| async move {
8454                let mut detached = (*state).clone();
8455                detached.count = 999;
8456                Ok(json!(detached.count))
8457            },
8458        );
8459        worker.register_replayed_query::<ReplayCounterState, _, _>(
8460            "replay-counter",
8461            "failed-mutation",
8462            |_ctx, state, _args| async move {
8463                let mut detached = (*state).clone();
8464                detached.count = 999;
8465                Err(Error::WorkerLoop("query refused".to_string()))
8466            },
8467        );
8468        worker
8469    }
8470
8471    fn replay_counter_query(
8472        query_name: &str,
8473        history_events: Value,
8474        run_status: &str,
8475    ) -> QueryTask {
8476        serde_json::from_value(json!({
8477            "query_task_id": format!("query-{query_name}"),
8478            "workflow_type": "replay-counter",
8479            "query_name": query_name,
8480            "payload_codec": "json",
8481            "workflow_arguments": {"codec": "json", "blob": "[]"},
8482            "query_arguments": {"codec": "json", "blob": "[]"},
8483            "history_events": history_events,
8484            "run_status": run_status,
8485        }))
8486        .expect("query task")
8487    }
8488
8489    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8490        workflow_context_with_codec(history, JSON_CODEC)
8491    }
8492
8493    fn workflow_context_with_codec(
8494        history: Vec<HistoryEvent>,
8495        payload_codec: &str,
8496    ) -> WorkflowContext {
8497        WorkflowContext {
8498            state: Arc::new(Mutex::new(
8499                WorkflowState::new_with_identity(
8500                    history,
8501                    None,
8502                    None,
8503                    "rust-workers".to_string(),
8504                    payload_codec.to_string(),
8505                    None,
8506                )
8507                .expect("valid workflow history"),
8508            )),
8509        }
8510    }
8511
8512    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8513        HistoryEvent {
8514            event_type: event_type.to_string(),
8515            payload,
8516            raw: HashMap::new(),
8517        }
8518    }
8519
8520    fn workflow_task(
8521        workflow_type: &str,
8522        history_events: Vec<HistoryEvent>,
8523        payload_codec: &str,
8524    ) -> WorkflowTask {
8525        WorkflowTask {
8526            task_id: format!("wft-{workflow_type}"),
8527            workflow_id: Some(format!("wf-{workflow_type}")),
8528            run_id: Some(format!("run-{workflow_type}")),
8529            workflow_type: workflow_type.to_string(),
8530            payload_codec: payload_codec.to_string(),
8531            arguments: Some(
8532                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8533            ),
8534            total_history_events: Some(history_events.len() as u64),
8535            history_size_bytes: None,
8536            continue_as_new_recommended: None,
8537            history_budget_pressure: None,
8538            history_events,
8539            next_history_page_token: None,
8540            workflow_task_attempt: 1,
8541            workflow_signal_id: None,
8542            signal_name: None,
8543            signal_arguments: None,
8544            workflow_update_id: None,
8545            update_name: None,
8546            lease_owner: Some("rust-worker".to_string()),
8547        }
8548    }
8549
8550    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8551    struct SideEffectProbe {
8552        request_id: String,
8553        attempt: u32,
8554    }
8555
8556    #[test]
8557    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8558        let calls = AtomicUsize::new(0);
8559        let ctx = workflow_context(Vec::new());
8560        let value = ctx
8561            .side_effect(|| {
8562                calls.fetch_add(1, Ordering::SeqCst);
8563                SideEffectProbe {
8564                    request_id: "request-42".to_string(),
8565                    attempt: 3,
8566                }
8567            })
8568            .expect("first side effect");
8569        assert_eq!(value.attempt, 3);
8570        assert_eq!(calls.load(Ordering::SeqCst), 1);
8571        let commands = ctx.take_commands().expect("commands");
8572        assert_eq!(commands.len(), 1);
8573        assert_eq!(commands[0]["type"], "record_side_effect");
8574        assert_eq!(
8575            decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8576            serde_json::to_value(&value).expect("value")
8577        );
8578
8579        let replay = workflow_context(vec![history_event(
8580            "SideEffectRecorded",
8581            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8582        )]);
8583        let replayed: SideEffectProbe = replay
8584            .side_effect(|| {
8585                calls.fetch_add(1, Ordering::SeqCst);
8586                panic!("committed side-effect callbacks must not run during replay")
8587            })
8588            .expect("replayed side effect");
8589        assert_eq!(replayed, value);
8590        assert_eq!(calls.load(Ordering::SeqCst), 1);
8591        assert!(replay.take_commands().expect("commands").is_empty());
8592        replay.ensure_history_consumed().expect("history consumed");
8593    }
8594
8595    #[test]
8596    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8597        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8598        let value = ctx
8599            .side_effect(|| SideEffectProbe {
8600                request_id: "avro-request".to_string(),
8601                attempt: 1,
8602            })
8603            .expect("Avro side effect");
8604        let uuid = ctx.uuid_v4().expect("deterministic UUID");
8605        let commands = ctx.take_commands().expect("commands");
8606        assert_eq!(commands.len(), 2);
8607        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8608        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8609        assert_eq!(
8610            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8611            serde_json::to_value(&value).expect("value")
8612        );
8613
8614        let replay = workflow_context_with_codec(
8615            vec![
8616                history_event(
8617                    "SideEffectRecorded",
8618                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8619                ),
8620                history_event(
8621                    "SideEffectRecorded",
8622                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8623                ),
8624            ],
8625            DEFAULT_CODEC,
8626        );
8627        let replayed: SideEffectProbe = replay
8628            .side_effect(|| panic!("Avro callback must not run"))
8629            .expect("replayed Avro value");
8630        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8631        assert_eq!(replayed, value);
8632        assert_eq!(replayed_uuid, uuid);
8633        assert!(replay.take_commands().expect("commands").is_empty());
8634    }
8635
8636    #[test]
8637    fn typed_side_effect_replay_preserves_bytes_and_maps() {
8638        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8639        let value = ctx
8640            .side_effect_avro_value(typed_fidelity_probe)
8641            .expect("typed side effect");
8642        let commands = ctx.take_commands().expect("side-effect command");
8643        assert_eq!(
8644            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8645                .expect("recorded side effect"),
8646            value
8647        );
8648
8649        let replay = workflow_context_with_codec(
8650            vec![history_event(
8651                "SideEffectRecorded",
8652                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8653            )],
8654            DEFAULT_CODEC,
8655        );
8656        assert_eq!(
8657            replay
8658                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8659                .expect("replayed typed side effect"),
8660            value
8661        );
8662    }
8663
8664    #[test]
8665    fn ordered_side_effects_share_the_durable_command_stream() {
8666        let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8667        let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8668        let ctx = workflow_context(vec![
8669            history_event(
8670                "SideEffectRecorded",
8671                json!({"sequence": 1, "result": first}),
8672            ),
8673            history_event(
8674                "SideEffectRecorded",
8675                json!({"sequence": 2, "result": second}),
8676            ),
8677        ]);
8678        let first: String = ctx
8679            .side_effect(|| panic!("first callback must not run"))
8680            .expect("first replay");
8681        let second: i32 = ctx
8682            .side_effect(|| panic!("second callback must not run"))
8683            .expect("second replay");
8684        assert_eq!(first, "first");
8685        assert_eq!(second, 29);
8686        ctx.ensure_history_consumed().expect("ordered history");
8687
8688        let reordered = workflow_context(vec![history_event(
8689            "VersionMarkerRecorded",
8690            json!({
8691                "sequence": 1,
8692                "change_id": "before-side-effect",
8693                "version": 1,
8694                "min_supported": 1,
8695                "max_supported": 1,
8696            }),
8697        )]);
8698        let error = reordered
8699            .side_effect(|| "new".to_string())
8700            .expect_err("command reordering must fail");
8701        assert!(matches!(
8702            error,
8703            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8704                if reason == "recorded_command_mismatch"
8705        ));
8706    }
8707
8708    #[test]
8709    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8710        let ctx = workflow_context(Vec::new());
8711        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8712        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8713        assert!(ctx.patched("new-search").expect("patch"));
8714        ctx.deprecate_patch("new-search").expect("deprecate patch");
8715        let commands = ctx.take_commands().expect("commands");
8716        assert_eq!(commands.len(), 2);
8717        assert_eq!(commands[0]["type"], "record_version_marker");
8718        assert_eq!(commands[0]["version"], 2);
8719        assert_eq!(commands[1]["change_id"], "new-search");
8720
8721        let replay = workflow_context(vec![history_event(
8722            "VersionMarkerRecorded",
8723            json!({
8724                "sequence": 1,
8725                "change_id": "checkout-v2",
8726                "version": 2,
8727                "min_supported": 1,
8728                "max_supported": 2,
8729            }),
8730        )]);
8731        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8732        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8733        assert!(replay.take_commands().expect("commands").is_empty());
8734        replay.ensure_history_consumed().expect("history consumed");
8735    }
8736
8737    #[test]
8738    fn version_markers_reject_incompatible_or_malformed_history() {
8739        let incompatible = workflow_context(vec![history_event(
8740            "VersionMarkerRecorded",
8741            json!({
8742                "sequence": 1,
8743                "change_id": "checkout-v2",
8744                "version": 1,
8745                "min_supported": 1,
8746                "max_supported": 2,
8747            }),
8748        )]);
8749        let error = incompatible
8750            .get_version("checkout-v2", 2, 3)
8751            .expect_err("old version is unsupported");
8752        assert!(matches!(
8753            error,
8754            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8755                if reason == "version_marker_incompatible_range"
8756        ));
8757
8758        for (history, reason) in [
8759            (
8760                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8761                "side_effect_result_missing",
8762            ),
8763            (
8764                vec![history_event(
8765                    "SideEffectRecorded",
8766                    json!({
8767                        "sequence": 1,
8768                        "result": {"codec": "avro", "blob": "not-base64"},
8769                    }),
8770                )],
8771                "side_effect_payload_incompatible",
8772            ),
8773            (
8774                vec![history_event(
8775                    "SideEffectRecorded",
8776                    json!({"sequence": 1, "result": {"unwrapped": true}}),
8777                )],
8778                "side_effect_payload_malformed",
8779            ),
8780            (
8781                vec![history_event(
8782                    "VersionMarkerRecorded",
8783                    json!({
8784                        "sequence": 1,
8785                        "change_id": "change",
8786                        "version": 1,
8787                        "min_supported": 2,
8788                        "max_supported": 1,
8789                    }),
8790                )],
8791                "version_marker_history_range_invalid",
8792            ),
8793        ] {
8794            let error = WorkflowState::new(
8795                history,
8796                "rust-workers".to_string(),
8797                JSON_CODEC.to_string(),
8798                None,
8799            )
8800            .expect_err("malformed history must fail");
8801            assert!(matches!(
8802                error,
8803                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8804                    if actual == reason
8805            ));
8806        }
8807    }
8808
8809    #[test]
8810    fn duplicate_side_effects_and_version_markers_are_rejected() {
8811        let duplicate_side_effect = WorkflowState::new(
8812            vec![
8813                history_event(
8814                    "SideEffectRecorded",
8815                    json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8816                ),
8817                history_event(
8818                    "SideEffectRecorded",
8819                    json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8820                ),
8821            ],
8822            "rust-workers".to_string(),
8823            JSON_CODEC.to_string(),
8824            None,
8825        )
8826        .expect_err("duplicate side effect");
8827        assert!(matches!(
8828            duplicate_side_effect,
8829            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8830                if reason == "duplicate_side_effect_record"
8831        ));
8832
8833        let marker = |sequence| {
8834            history_event(
8835                "VersionMarkerRecorded",
8836                json!({
8837                    "sequence": sequence,
8838                    "change_id": "same-change",
8839                    "version": 1,
8840                    "min_supported": 1,
8841                    "max_supported": 1,
8842                }),
8843            )
8844        };
8845        let duplicate_marker = WorkflowState::new(
8846            vec![marker(1), marker(3)],
8847            "rust-workers".to_string(),
8848            JSON_CODEC.to_string(),
8849            None,
8850        )
8851        .expect_err("duplicate marker");
8852        assert!(matches!(
8853            duplicate_marker,
8854            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8855                if reason == "duplicate_version_marker"
8856        ));
8857    }
8858
8859    #[test]
8860    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8861        fn worker(calls: Arc<AtomicUsize>) -> Worker {
8862            let client = Client::new("http://127.0.0.1:8080").expect("client");
8863            let mut worker = Worker::new(client, "rust-workers");
8864            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8865                let calls = Arc::clone(&calls);
8866                async move {
8867                    let captured = ctx.side_effect(|| {
8868                        calls.fetch_add(1, Ordering::SeqCst);
8869                        "captured-once".to_string()
8870                    })?;
8871                    let version = ctx.get_version("cold-restart", 1, 2)?;
8872                    Ok(json!({"captured": captured, "version": version}))
8873                }
8874            });
8875            worker
8876        }
8877
8878        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8879            WorkflowTask {
8880                task_id: "wft-side-effect-version".to_string(),
8881                workflow_id: Some("wf-side-effect-version".to_string()),
8882                run_id: Some("run-side-effect-version".to_string()),
8883                workflow_type: "rust.side-effect-version".to_string(),
8884                payload_codec: JSON_CODEC.to_string(),
8885                arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8886                history_events,
8887                total_history_events: None,
8888                history_size_bytes: None,
8889                continue_as_new_recommended: None,
8890                history_budget_pressure: None,
8891                next_history_page_token: None,
8892                workflow_task_attempt: 1,
8893                workflow_signal_id: None,
8894                signal_name: None,
8895                signal_arguments: None,
8896                workflow_update_id: None,
8897                update_name: None,
8898                lease_owner: Some("rust-worker".to_string()),
8899            }
8900        }
8901
8902        let calls = Arc::new(AtomicUsize::new(0));
8903        let initial = worker(Arc::clone(&calls))
8904            .execute_workflow_task(task(Vec::new()))
8905            .expect("initial execution");
8906        assert_eq!(
8907            initial
8908                .iter()
8909                .map(|command| &command["type"])
8910                .collect::<Vec<_>>(),
8911            vec![
8912                "record_side_effect",
8913                "record_version_marker",
8914                "complete_workflow"
8915            ]
8916        );
8917        assert_eq!(calls.load(Ordering::SeqCst), 1);
8918
8919        let restarted = worker(Arc::clone(&calls));
8920        let replayed = restarted
8921            .execute_workflow_task(task(vec![
8922                history_event(
8923                    "SideEffectRecorded",
8924                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8925                ),
8926                history_event(
8927                    "VersionMarkerRecorded",
8928                    json!({
8929                        "sequence": 2,
8930                        "change_id": "cold-restart",
8931                        "version": 2,
8932                        "min_supported": 1,
8933                        "max_supported": 2,
8934                    }),
8935                ),
8936            ]))
8937            .expect("cold replay");
8938        assert_eq!(replayed.len(), 1);
8939        assert_eq!(replayed[0]["type"], "complete_workflow");
8940        assert_eq!(calls.load(Ordering::SeqCst), 1);
8941    }
8942
8943    #[test]
8944    fn side_effect_replay_rejects_changed_rust_value_type() {
8945        let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8946        let ctx = workflow_context(vec![history_event(
8947            "SideEffectRecorded",
8948            json!({"sequence": 1, "result": result}),
8949        )]);
8950        let error = ctx
8951            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8952            .expect_err("changed type must fail replay");
8953        assert!(matches!(
8954            error,
8955            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8956                if reason == "side_effect_type_mismatch"
8957        ));
8958    }
8959
8960    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8961        vec![
8962            history_event(
8963                "ActivityScheduled",
8964                json!({
8965                    "sequence": 1,
8966                    "activity_type": "flaky",
8967                    "activity_execution_id": "act-1",
8968                    "activity": {
8969                        "id": "act-1",
8970                        "sequence": 1,
8971                        "type": "flaky",
8972                        "queue": "critical-activities",
8973                        "execution_mode": null,
8974                        "retry_policy": {
8975                            "snapshot_version": 1,
8976                            "max_attempts": 3,
8977                            "backoff_seconds": [2, 4],
8978                            "start_to_close_timeout": 30,
8979                            "schedule_to_start_timeout": 5,
8980                            "schedule_to_close_timeout": 90,
8981                            "heartbeat_timeout": 10,
8982                            "non_retryable_error_types": ["PermanentError"]
8983                        }
8984                    }
8985                }),
8986            ),
8987            history_event(
8988                "ActivityStarted",
8989                json!({
8990                    "sequence": 1,
8991                    "activity_type": "flaky",
8992                    "activity_execution_id": "act-1",
8993                    "activity_attempt_id": "attempt-1",
8994                    "attempt_number": 1
8995                }),
8996            ),
8997            history_event(
8998                "ActivityRetryScheduled",
8999                json!({
9000                    "sequence": 1,
9001                    "activity_type": "flaky",
9002                    "activity_execution_id": "act-1",
9003                    "activity_attempt_id": "attempt-1",
9004                    "attempt_number": 1,
9005                    "retry_after_attempt": 1,
9006                    "retry_backoff_seconds": 2,
9007                    "failure_category": "activity",
9008                    "exception_type": "TransientError"
9009                }),
9010            ),
9011            history_event(
9012                "ActivityStarted",
9013                json!({
9014                    "sequence": 1,
9015                    "activity_type": "flaky",
9016                    "activity_execution_id": "act-1",
9017                    "activity_attempt_id": "attempt-2",
9018                    "attempt_number": 2
9019                }),
9020            ),
9021            history_event(
9022                "ActivityCompleted",
9023                json!({
9024                    "sequence": 1,
9025                    "activity_type": "flaky",
9026                    "activity_execution_id": "act-1",
9027                    "activity_attempt_id": "attempt-2",
9028                    "attempt_number": 2,
9029                    "payload_codec": "json",
9030                    "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
9031                }),
9032            ),
9033        ]
9034    }
9035
9036    fn retry_activity_options() -> ActivityOptions {
9037        ActivityOptions::new()
9038            .task_queue("critical-activities")
9039            .retry_policy(
9040                ActivityRetryPolicy::new(3)
9041                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9042                    .non_retryable_error_type("PermanentError"),
9043            )
9044            .start_to_close_timeout(Duration::from_secs(30))
9045            .schedule_to_start_timeout(Duration::from_secs(5))
9046            .schedule_to_close_timeout(Duration::from_secs(90))
9047            .heartbeat_timeout(Duration::from_secs(10))
9048    }
9049
9050    #[test]
9051    fn fixed_avro_value_round_trips_json_values() {
9052        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9053        let envelope = PayloadEnvelope::avro(&value).expect("encode");
9054        assert_eq!(envelope.codec, DEFAULT_CODEC);
9055        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9056    }
9057
9058    #[tokio::test]
9059    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9060        let client = Client::new("http://127.0.0.1:8080").expect("client");
9061        let mut worker = Worker::new(client, "rust-workers");
9062        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9063        worker
9064            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9065        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9066            Ok(input)
9067        });
9068        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9069            Ok(input)
9070        });
9071        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9072            Ok(AvroValue::Array(
9073                ctx.wait_signal_avro_value("changed").await?,
9074            ))
9075        });
9076
9077        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9078        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9079
9080        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9081        workflow.arguments = Some(envelope.clone());
9082        let commands = worker
9083            .execute_workflow_task(workflow)
9084            .expect("typed workflow task");
9085        assert_eq!(commands[0]["type"], "complete_workflow");
9086        assert_eq!(
9087            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9088                .expect("typed workflow result"),
9089            arguments
9090        );
9091
9092        let activity = ActivityTask {
9093            task_id: "activity-typed".to_string(),
9094            activity_attempt_id: Some("attempt-typed".to_string()),
9095            attempt_id: None,
9096            activity_type: "typed.activity".to_string(),
9097            payload_codec: DEFAULT_CODEC.to_string(),
9098            arguments: Some(envelope.clone()),
9099            attempt_number: 1,
9100            lease_owner: Some("rust-worker".to_string()),
9101        };
9102        assert_eq!(
9103            worker
9104                .execute_activity_task(activity)
9105                .await
9106                .expect("typed activity result"),
9107            arguments
9108        );
9109
9110        let query = QueryTask {
9111            query_task_id: "query-typed".to_string(),
9112            query_task_attempt: 1,
9113            lease_owner: Some("rust-worker".to_string()),
9114            workflow_id: Some("typed-1".to_string()),
9115            run_id: Some("run-typed".to_string()),
9116            workflow_type: "typed.echo".to_string(),
9117            query_name: "inspect".to_string(),
9118            payload_codec: DEFAULT_CODEC.to_string(),
9119            workflow_arguments: Some(
9120                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9121                    .expect("workflow input"),
9122            ),
9123            query_arguments: Some(envelope.clone()),
9124            history_events: Vec::new(),
9125            history_export: None,
9126            run_status: Some("running".to_string()),
9127        };
9128        assert_eq!(
9129            worker
9130                .execute_query_task(query)
9131                .await
9132                .expect("typed query result"),
9133            arguments
9134        );
9135
9136        let mut update = workflow_task(
9137            "typed.echo",
9138            vec![history_event(
9139                "UpdateAccepted",
9140                json!({
9141                    "update_id": "update-typed",
9142                    "update_name": "replace",
9143                    "arguments": envelope.clone(),
9144                }),
9145            )],
9146            DEFAULT_CODEC,
9147        );
9148        update.workflow_update_id = Some("update-typed".to_string());
9149        update.update_name = Some("replace".to_string());
9150        let commands = worker
9151            .execute_workflow_task(update)
9152            .expect("typed update task");
9153        assert_eq!(commands[0]["type"], "complete_update");
9154        assert_eq!(
9155            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9156                .expect("typed update result"),
9157            arguments
9158        );
9159
9160        let mut signal = workflow_task(
9161            "typed.signal",
9162            vec![history_event(
9163                "SignalReceived",
9164                json!({
9165                    "signal_id": "signal-typed",
9166                    "signal_name": "changed",
9167                    "arguments": envelope.clone(),
9168                }),
9169            )],
9170            DEFAULT_CODEC,
9171        );
9172        signal.workflow_signal_id = Some("signal-typed".to_string());
9173        signal.signal_name = Some("changed".to_string());
9174        signal.signal_arguments = Some(envelope);
9175        let commands = worker
9176            .execute_workflow_task(signal)
9177            .expect("typed signal resume");
9178        assert_eq!(
9179            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9180                .expect("typed signal result"),
9181            arguments
9182        );
9183    }
9184
9185    #[tokio::test]
9186    async fn typed_helpers_never_parse_json_inspection_projection() {
9187        let collision_values = projection_collision_probe();
9188        let expected = AvroValue::Array(collision_values.clone());
9189        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9190
9191        let activity_context = workflow_context_with_codec(
9192            vec![history_event(
9193                "ActivityCompleted",
9194                json!({
9195                    "sequence": 1,
9196                    "activity_type": "collision.activity",
9197                    "payload_codec": DEFAULT_CODEC,
9198                    "result": envelope.clone(),
9199                }),
9200            )],
9201            DEFAULT_CODEC,
9202        );
9203        assert_eq!(
9204            activity_context
9205                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9206                .await
9207                .expect("typed activity collision result"),
9208            expected
9209        );
9210
9211        let signal_context = workflow_context_with_codec(
9212            vec![
9213                history_event(
9214                    "SignalWaitOpened",
9215                    json!({"sequence": 1, "signal_name": "collision"}),
9216                ),
9217                history_event(
9218                    "SignalApplied",
9219                    json!({
9220                        "sequence": 1,
9221                        "signal_name": "collision",
9222                        "payload_codec": DEFAULT_CODEC,
9223                        "value": envelope.clone(),
9224                    }),
9225                ),
9226            ],
9227            DEFAULT_CODEC,
9228        );
9229        assert_eq!(
9230            signal_context
9231                .wait_signal_avro_value("collision")
9232                .await
9233                .expect("typed signal collision arguments"),
9234            collision_values
9235        );
9236
9237        let child_context = workflow_context_with_codec(
9238            vec![
9239                history_event(
9240                    "ChildWorkflowScheduled",
9241                    json!({
9242                        "sequence": 1,
9243                        "child_workflow_instance_id": "collision-child",
9244                        "child_workflow_run_id": "collision-run",
9245                        "child_workflow_type": "collision.child",
9246                    }),
9247                ),
9248                history_event(
9249                    "ChildRunCompleted",
9250                    json!({
9251                        "sequence": 1,
9252                        "child_workflow_instance_id": "collision-child",
9253                        "child_workflow_run_id": "collision-run",
9254                        "child_workflow_type": "collision.child",
9255                        "payload_codec": DEFAULT_CODEC,
9256                        "result": envelope,
9257                    }),
9258                ),
9259            ],
9260            DEFAULT_CODEC,
9261        );
9262        let child = child_context
9263            .start_child_workflow_avro_value(
9264                "collision.child",
9265                ChildWorkflowOptions::new("collision-workers"),
9266                AvroValue::Array(Vec::new()),
9267            )
9268            .await
9269            .expect("typed child collision result");
9270        assert_eq!(child.result, expected);
9271    }
9272
9273    #[tokio::test]
9274    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9275        let client = Client::new("http://127.0.0.1:8080").expect("client");
9276        let mut worker = Worker::new(client, "rust-workers");
9277        worker.register_replayed_workflow_avro_value(
9278            "typed.replayed",
9279            || (),
9280            |_ctx, input, _state| async move { Ok(input) },
9281        );
9282        worker.register_replayed_query_avro_value::<(), _, _>(
9283            "typed.replayed",
9284            "inspect",
9285            |ctx, _state, args| async move {
9286                let mut signals = ctx.signals_avro_value("collision");
9287                let signal = signals
9288                    .pop()
9289                    .map(AvroValue::Array)
9290                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9291                Ok(AvroValue::Array(vec![
9292                    ctx.workflow_input_avro_value().clone(),
9293                    signal,
9294                    args,
9295                ]))
9296            },
9297        );
9298        let arguments = AvroValue::Array(projection_collision_probe());
9299        let signal_arguments =
9300            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9301        let task = QueryTask {
9302            query_task_id: "query-typed-replay".to_string(),
9303            query_task_attempt: 1,
9304            lease_owner: Some("rust-worker".to_string()),
9305            workflow_id: Some("typed-replay".to_string()),
9306            run_id: Some("run-typed-replay".to_string()),
9307            workflow_type: "typed.replayed".to_string(),
9308            query_name: "inspect".to_string(),
9309            payload_codec: DEFAULT_CODEC.to_string(),
9310            workflow_arguments: Some(
9311                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9312            ),
9313            query_arguments: Some(
9314                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9315            ),
9316            history_events: vec![history_event(
9317                "SignalReceived",
9318                json!({
9319                    "signal_id": "collision-signal",
9320                    "signal_name": "collision",
9321                    "workflow_sequence": 1,
9322                    "payload_codec": DEFAULT_CODEC,
9323                    "arguments": signal_arguments,
9324                }),
9325            )],
9326            history_export: None,
9327            run_status: Some("completed".to_string()),
9328        };
9329
9330        assert_eq!(
9331            worker
9332                .execute_query_task(task)
9333                .await
9334                .expect("typed replay query"),
9335            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9336        );
9337    }
9338
9339    #[test]
9340    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9341        let value = BTreeMap::from([(1_i32, "integer key")]);
9342        let error = PayloadEnvelope::avro(&value)
9343            .expect_err("integer map keys must fail")
9344            .to_string();
9345
9346        assert!(error.contains("invalid_map_key"));
9347    }
9348
9349    #[test]
9350    fn json_codec_remains_plain_json() {
9351        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9352        let envelope = PayloadEnvelope::json(&value).expect("encode");
9353
9354        assert_eq!(envelope.codec, JSON_CODEC);
9355        assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9356        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9357    }
9358
9359    #[test]
9360    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9361        let envelope = PayloadEnvelope {
9362            codec: DEFAULT_CODEC.to_string(),
9363            blob: BASE64.encode([0x01]),
9364        };
9365
9366        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9367        assert!(error.to_string().contains("invalid_payload_framing"));
9368    }
9369
9370    #[test]
9371    fn workflow_context_schedules_activity_until_completion_is_in_history() {
9372        let ctx = WorkflowContext {
9373            state: Arc::new(Mutex::new(
9374                WorkflowState::new_with_identity(
9375                    Vec::new(),
9376                    Some("wf-parent".to_string()),
9377                    Some("run-parent".to_string()),
9378                    "rust-workers".to_string(),
9379                    DEFAULT_CODEC.to_string(),
9380                    None,
9381                )
9382                .expect("workflow state"),
9383            )),
9384        };
9385
9386        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9387        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9388        assert!(matches!(
9389            call.as_mut().poll(&mut task_context),
9390            Poll::Pending
9391        ));
9392
9393        let commands = ctx.take_commands().expect("commands");
9394        assert_eq!(commands[0]["type"], "schedule_activity");
9395        assert_eq!(commands[0]["activity_type"], "hello.activity");
9396    }
9397
9398    #[test]
9399    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9400        let ctx = workflow_context(Vec::new());
9401        let options = ActivityOptions::new()
9402            .task_queue("payments")
9403            .retry_policy(
9404                ActivityRetryPolicy::new(4)
9405                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9406                    .non_retryable_error_type("ValidationError"),
9407            )
9408            .start_to_close_timeout(Duration::from_secs(120))
9409            .schedule_to_start_timeout(Duration::from_secs(10))
9410            .schedule_to_close_timeout(Duration::from_secs(300))
9411            .heartbeat_timeout(Duration::from_secs(15));
9412        let mut call = Box::pin(ctx.activity_with_options(
9413            "charge-card",
9414            options,
9415            json!([{"order_id": "o-1"}]),
9416        ));
9417        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9418
9419        assert!(matches!(
9420            call.as_mut().poll(&mut task_context),
9421            Poll::Pending
9422        ));
9423        assert!(matches!(
9424            call.as_mut().poll(&mut task_context),
9425            Poll::Pending
9426        ));
9427
9428        let commands = ctx.take_commands().expect("activity command");
9429        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9430        assert_eq!(commands[0]["queue"], "payments");
9431        assert_eq!(
9432            commands[0]["retry_policy"],
9433            json!({
9434                "max_attempts": 4,
9435                "backoff_seconds": [1, 3, 9],
9436                "non_retryable_error_types": ["ValidationError"],
9437            })
9438        );
9439        assert_eq!(commands[0]["start_to_close_timeout"], 120);
9440        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9441        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9442        assert_eq!(commands[0]["heartbeat_timeout"], 15);
9443    }
9444
9445    #[test]
9446    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9447        let ctx = workflow_context(Vec::new());
9448        let options = ActivityOptions::new().retry_policy(
9449            ActivityRetryPolicy::new(3)
9450                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9451        );
9452        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9453        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9454
9455        assert!(matches!(
9456            call.as_mut().poll(&mut task_context),
9457            Poll::Pending
9458        ));
9459        assert_eq!(
9460            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9461            json!([1, 2])
9462        );
9463    }
9464
9465    #[test]
9466    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9467        let cases = [
9468            (
9469                ActivityOptions::new().task_queue("  "),
9470                ActivityOptionsErrorKind::EmptyTaskQueue,
9471            ),
9472            (
9473                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9474                ActivityOptionsErrorKind::EmptyRetryPolicy,
9475            ),
9476            (
9477                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9478                ActivityOptionsErrorKind::InvalidMaxAttempts,
9479            ),
9480            (
9481                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9482                    max_attempts: None,
9483                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9484                    non_retryable_error_types: Vec::new(),
9485                }),
9486                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9487            ),
9488            (
9489                ActivityOptions::new().retry_policy(
9490                    ActivityRetryPolicy::new(2)
9491                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9492                ),
9493                ActivityOptionsErrorKind::TooManyBackoffIntervals,
9494            ),
9495            (
9496                ActivityOptions::new().retry_policy(
9497                    ActivityRetryPolicy::new(2).exponential_backoff(
9498                        Duration::from_secs(1),
9499                        0,
9500                        None,
9501                    ),
9502                ),
9503                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9504            ),
9505            (
9506                ActivityOptions::new()
9507                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
9508                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9509            ),
9510            (
9511                ActivityOptions::new().retry_policy(
9512                    ActivityRetryPolicy::new(10_002).exponential_backoff(
9513                        Duration::from_secs(1),
9514                        1,
9515                        None,
9516                    ),
9517                ),
9518                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9519            ),
9520            (
9521                ActivityOptions::new().retry_policy(
9522                    ActivityRetryPolicy::new(2)
9523                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9524                ),
9525                ActivityOptionsErrorKind::BackoffOverflow,
9526            ),
9527        ];
9528
9529        for (options, expected_kind) in cases {
9530            let ctx = workflow_context(Vec::new());
9531            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9532            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9533            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9534                call.as_mut().poll(&mut task_context)
9535            else {
9536                panic!("expected typed activity validation error");
9537            };
9538            assert_eq!(error.kind, expected_kind);
9539            assert!(ctx.take_commands().expect("commands").is_empty());
9540        }
9541    }
9542
9543    #[test]
9544    fn activity_options_validate_positive_and_ordered_timeouts() {
9545        let zero_timeout_cases = [
9546            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9547            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9548            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9549            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9550        ];
9551        for options in zero_timeout_cases {
9552            assert_eq!(
9553                options.validate().expect_err("zero timeout").kind,
9554                ActivityOptionsErrorKind::TimeoutNotPositive
9555            );
9556        }
9557
9558        let ordering_cases = [
9559            ActivityOptions::new()
9560                .heartbeat_timeout(Duration::from_secs(11))
9561                .start_to_close_timeout(Duration::from_secs(10)),
9562            ActivityOptions::new()
9563                .start_to_close_timeout(Duration::from_secs(31))
9564                .schedule_to_close_timeout(Duration::from_secs(30)),
9565            ActivityOptions::new()
9566                .schedule_to_start_timeout(Duration::from_secs(31))
9567                .schedule_to_close_timeout(Duration::from_secs(30)),
9568        ];
9569        for options in ordering_cases {
9570            assert_eq!(
9571                options.validate().expect_err("timeout order").kind,
9572                ActivityOptionsErrorKind::TimeoutOrder
9573            );
9574        }
9575
9576        assert_eq!(
9577            ActivityOptions::new()
9578                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9579                .validate()
9580                .expect_err("protocol integer overflow")
9581                .kind,
9582            ActivityOptionsErrorKind::TimeoutOverflow
9583        );
9584    }
9585
9586    #[test]
9587    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9588        let ctx = workflow_context(completed_retry_activity_history());
9589        let mut call =
9590            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9591        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9592
9593        assert!(matches!(
9594            call.as_mut().poll(&mut task_context),
9595            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9596        ));
9597        assert!(ctx.take_commands().expect("commands").is_empty());
9598        ctx.ensure_history_consumed().expect("history consumed");
9599    }
9600
9601    #[test]
9602    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9603        let mut options = retry_activity_options();
9604        options
9605            .retry_policy
9606            .as_mut()
9607            .expect("retry policy")
9608            .non_retryable_error_types
9609            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9610
9611        let new_ctx = workflow_context(Vec::new());
9612        let mut new_call =
9613            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9614        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9615        assert!(matches!(
9616            new_call.as_mut().poll(&mut task_context),
9617            Poll::Pending
9618        ));
9619        let commands = new_ctx.take_commands().expect("commands");
9620        assert_eq!(commands.len(), 1);
9621        assert_eq!(
9622            commands[0]["retry_policy"]["non_retryable_error_types"],
9623            json!(["PermanentError"])
9624        );
9625
9626        let replay_ctx = workflow_context(completed_retry_activity_history());
9627        let mut replay_call =
9628            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9629        assert!(matches!(
9630            replay_call.as_mut().poll(&mut task_context),
9631            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9632        ));
9633        assert!(replay_ctx.take_commands().expect("commands").is_empty());
9634        replay_ctx
9635            .ensure_history_consumed()
9636            .expect("history consumed");
9637    }
9638
9639    #[test]
9640    fn replayed_intermediate_retry_remains_pending_across_restarts() {
9641        let history = completed_retry_activity_history()
9642            .into_iter()
9643            .take(3)
9644            .collect::<Vec<_>>();
9645
9646        for _restart in 0..2 {
9647            let ctx = workflow_context(history.clone());
9648            let mut call =
9649                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9650            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9651            assert!(matches!(
9652                call.as_mut().poll(&mut task_context),
9653                Poll::Pending
9654            ));
9655            assert!(ctx.take_commands().expect("commands").is_empty());
9656        }
9657    }
9658
9659    #[test]
9660    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9661        let mut changed_queue = retry_activity_options();
9662        changed_queue.task_queue = Some("different-queue".to_string());
9663
9664        let mut changed_max_attempts = retry_activity_options();
9665        let retry_policy = changed_max_attempts
9666            .retry_policy
9667            .as_mut()
9668            .expect("retry policy");
9669        retry_policy.max_attempts = Some(4);
9670
9671        let mut changed_backoff = retry_activity_options();
9672        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9673        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9674            Duration::from_secs(3),
9675            Duration::from_secs(4),
9676        ]));
9677
9678        let mut changed_non_retryable_types = retry_activity_options();
9679        let retry_policy = changed_non_retryable_types
9680            .retry_policy
9681            .as_mut()
9682            .expect("retry policy");
9683        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9684
9685        let mut changed_start_to_close = retry_activity_options();
9686        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9687        let mut changed_schedule_to_start = retry_activity_options();
9688        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9689        let mut changed_schedule_to_close = retry_activity_options();
9690        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9691        let mut changed_heartbeat = retry_activity_options();
9692        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9693
9694        let cases = [
9695            (changed_queue, "activity_task_queue_mismatch"),
9696            (changed_max_attempts, "activity_retry_policy_mismatch"),
9697            (changed_backoff, "activity_retry_policy_mismatch"),
9698            (
9699                changed_non_retryable_types,
9700                "activity_retry_policy_mismatch",
9701            ),
9702            (changed_start_to_close, "activity_retry_policy_mismatch"),
9703            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9704            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9705            (changed_heartbeat, "activity_retry_policy_mismatch"),
9706        ];
9707
9708        for (options, expected_reason) in cases {
9709            let ctx = workflow_context(completed_retry_activity_history());
9710            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9711            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9712            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9713                call.as_mut().poll(&mut task_context)
9714            else {
9715                panic!("changed activity options must fail replay");
9716            };
9717            assert_eq!(failure.reason, expected_reason);
9718            assert_eq!(failure.sequence, Some(1));
9719            assert!(ctx.take_commands().expect("commands").is_empty());
9720        }
9721    }
9722
9723    #[test]
9724    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9725        let cases = [
9726            (
9727                "execution_mode",
9728                json!("local"),
9729                "activity_execution_mode_mismatch",
9730            ),
9731            (
9732                "snapshot_version",
9733                json!(2),
9734                "activity_retry_policy_mismatch",
9735            ),
9736        ];
9737
9738        for (field, value, expected_reason) in cases {
9739            let mut history = completed_retry_activity_history();
9740            let activity = history[0].payload["activity"]
9741                .as_object_mut()
9742                .expect("activity snapshot");
9743            if field == "execution_mode" {
9744                activity.insert(field.to_string(), value);
9745            } else {
9746                activity["retry_policy"]
9747                    .as_object_mut()
9748                    .expect("retry snapshot")
9749                    .insert(field.to_string(), value);
9750            }
9751
9752            let ctx = workflow_context(history);
9753            let mut call =
9754                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9755            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9756            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9757                call.as_mut().poll(&mut task_context)
9758            else {
9759                panic!("changed {field} must fail replay");
9760            };
9761            assert_eq!(failure.reason, expected_reason);
9762            assert_eq!(failure.sequence, Some(1));
9763            assert!(ctx.take_commands().expect("commands").is_empty());
9764        }
9765    }
9766
9767    #[test]
9768    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9769        let mut history = completed_retry_activity_history();
9770        let activity = history[0].payload["activity"]
9771            .as_object_mut()
9772            .expect("activity snapshot");
9773        activity.remove("execution_mode");
9774        activity.remove("retry_policy");
9775
9776        let mut current = retry_activity_options();
9777        current.start_to_close_timeout = Some(Duration::from_secs(45));
9778        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9779        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9780        current.heartbeat_timeout = Some(Duration::from_secs(12));
9781
9782        let ctx = workflow_context(history);
9783        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9784        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9785        assert!(matches!(
9786            call.as_mut().poll(&mut task_context),
9787            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9788        ));
9789        assert!(ctx.take_commands().expect("commands").is_empty());
9790        ctx.ensure_history_consumed().expect("history consumed");
9791    }
9792
9793    #[test]
9794    fn terminal_activity_failed_after_start_returns_typed_failure() {
9795        let history = vec![
9796            history_event(
9797                "ActivityScheduled",
9798                json!({
9799                    "sequence": 1,
9800                    "activity_type": "flaky",
9801                    "activity_execution_id": "act-terminal",
9802                    "activity": {
9803                        "id": "act-terminal",
9804                        "sequence": 1,
9805                        "type": "flaky",
9806                        "queue": "critical-activities",
9807                        "retry_policy": {
9808                            "snapshot_version": 1,
9809                            "max_attempts": 3,
9810                            "backoff_seconds": [2, 4],
9811                            "non_retryable_error_types": ["PermanentError"]
9812                        }
9813                    }
9814                }),
9815            ),
9816            history_event(
9817                "ActivityStarted",
9818                json!({
9819                    "sequence": 1,
9820                    "activity_type": "flaky",
9821                    "activity_execution_id": "act-terminal",
9822                    "activity_attempt_id": "attempt-1",
9823                    "attempt_number": 1
9824                }),
9825            ),
9826            history_event(
9827                "ActivityFailed",
9828                json!({
9829                    "sequence": 1,
9830                    "activity_type": "flaky",
9831                    "activity_execution_id": "act-terminal",
9832                    "activity_attempt_id": "attempt-1",
9833                    "attempt_number": 1,
9834                    "failure_id": "failure-terminal",
9835                    "failure_category": "activity",
9836                    "exception_type": "PermanentError",
9837                    "message": "cannot retry",
9838                    "non_retryable": true
9839                }),
9840            ),
9841        ];
9842        let ctx = workflow_context(history);
9843        let mut call =
9844            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9845        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9846
9847        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9848            call.as_mut().poll(&mut task_context)
9849        else {
9850            panic!("terminal ActivityFailed must settle the activity future");
9851        };
9852        assert_eq!(failure.kind, ActivityFailureKind::Failed);
9853        assert_eq!(
9854            failure.activity_execution_id.as_deref(),
9855            Some("act-terminal")
9856        );
9857        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9858        assert!(failure.non_retryable);
9859        assert!(ctx.take_commands().expect("commands").is_empty());
9860        ctx.ensure_history_consumed().expect("history consumed");
9861    }
9862
9863    #[test]
9864    fn activity_terminal_events_return_machine_readable_failures() {
9865        let cases = [
9866            (
9867                "ActivityFailed",
9868                json!({
9869                    "sequence": 1,
9870                    "activity_type": "charge-card",
9871                    "activity_execution_id": "act-1",
9872                    "activity_attempt_id": "attempt-2",
9873                    "attempt_number": 2,
9874                    "failure_id": "failure-1",
9875                    "failure_category": "activity",
9876                    "exception_type": "PaymentDeclined",
9877                    "exception_class": "payments.PaymentDeclined",
9878                    "message": "card declined",
9879                    "non_retryable": true
9880                }),
9881                ActivityFailureKind::Failed,
9882                "activity",
9883            ),
9884            (
9885                "ActivityCancelled",
9886                json!({
9887                    "sequence": 1,
9888                    "activity_type": "charge-card",
9889                    "activity_execution_id": "act-1",
9890                    "activity_attempt_id": "attempt-1"
9891                }),
9892                ActivityFailureKind::Cancelled,
9893                "cancelled",
9894            ),
9895        ];
9896
9897        for (event_type, payload, expected_kind, expected_reason) in cases {
9898            let ctx = workflow_context(vec![history_event(event_type, payload)]);
9899            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9900            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9901            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9902                call.as_mut().poll(&mut task_context)
9903            else {
9904                panic!("expected terminal activity failure");
9905            };
9906            assert_eq!(failure.kind, expected_kind);
9907            assert_eq!(failure.reason, expected_reason);
9908            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9909            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9910        }
9911    }
9912
9913    #[test]
9914    fn every_activity_timeout_class_is_typed() {
9915        for timeout_kind in [
9916            "start_to_close",
9917            "schedule_to_start",
9918            "schedule_to_close",
9919            "heartbeat",
9920        ] {
9921            let ctx = workflow_context(vec![history_event(
9922                "ActivityTimedOut",
9923                json!({
9924                    "sequence": 1,
9925                    "activity_type": "slow",
9926                    "activity_execution_id": "act-timeout",
9927                    "activity_attempt_id": "attempt-timeout",
9928                    "failure_category": "timeout",
9929                    "timeout_kind": timeout_kind,
9930                    "message": "deadline expired"
9931                }),
9932            )]);
9933            let mut call = Box::pin(ctx.activity("slow", json!([])));
9934            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9935            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9936                call.as_mut().poll(&mut task_context)
9937            else {
9938                panic!("expected timeout failure");
9939            };
9940            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9941            assert_eq!(failure.reason, timeout_kind);
9942            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9943            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9944        }
9945    }
9946
9947    #[test]
9948    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9949        let ctx = workflow_context(Vec::new());
9950        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9951        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9952
9953        assert!(matches!(
9954            sleep.as_mut().poll(&mut task_context),
9955            Poll::Pending
9956        ));
9957        assert!(matches!(
9958            sleep.as_mut().poll(&mut task_context),
9959            Poll::Pending
9960        ));
9961
9962        let commands = ctx.take_commands().expect("timer command");
9963        assert_eq!(
9964            commands,
9965            vec![json!({
9966                "type": "start_timer",
9967                "delay_seconds": 2,
9968            })]
9969        );
9970    }
9971
9972    #[test]
9973    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9974        let history = vec![
9975            history_event(
9976                "TimerScheduled",
9977                json!({
9978                    "sequence": 1,
9979                    "timer_id": "timer-1",
9980                    "delay_seconds": 5,
9981                    "fire_at": "2026-07-11T12:00:05Z",
9982                }),
9983            ),
9984            history_event(
9985                "TimerFired",
9986                json!({
9987                    "sequence": 1,
9988                    "timer_id": "timer-1",
9989                    "delay_seconds": 5,
9990                    "fire_at": "2026-07-11T12:00:05Z",
9991                    "fired_at": "2026-07-11T12:00:05Z",
9992                }),
9993            ),
9994        ];
9995
9996        for _restart in 0..2 {
9997            let ctx = workflow_context(history.clone());
9998            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
9999            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10000            assert!(matches!(
10001                sleep.as_mut().poll(&mut task_context),
10002                Poll::Ready(Ok(()))
10003            ));
10004            assert!(ctx.take_commands().expect("commands").is_empty());
10005            ctx.ensure_history_consumed().expect("history consumed");
10006        }
10007    }
10008
10009    #[test]
10010    fn workflow_sleep_rejects_changed_delay_during_replay() {
10011        let ctx = workflow_context(vec![
10012            history_event(
10013                "TimerScheduled",
10014                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10015            ),
10016            history_event(
10017                "TimerFired",
10018                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10019            ),
10020        ]);
10021        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10022        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10023
10024        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10025            sleep.as_mut().poll(&mut task_context)
10026        else {
10027            panic!("changed timer delay must be rejected");
10028        };
10029        assert_eq!(failure.reason, "timer_delay_mismatch");
10030        assert_eq!(failure.sequence, Some(1));
10031    }
10032
10033    #[test]
10034    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10035        let lone_fire = WorkflowState::new(
10036            vec![history_event(
10037                "TimerFired",
10038                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10039            )],
10040            "rust-workers".to_string(),
10041            JSON_CODEC.to_string(),
10042            None,
10043        )
10044        .expect_err("TimerFired requires TimerScheduled");
10045        assert!(matches!(
10046            lone_fire,
10047            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10048                if reason == "timer_schedule_missing_or_duplicate"
10049        ));
10050
10051        let wrong_identity = WorkflowState::new(
10052            vec![
10053                history_event(
10054                    "TimerScheduled",
10055                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10056                ),
10057                history_event(
10058                    "TimerFired",
10059                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10060                ),
10061            ],
10062            "rust-workers".to_string(),
10063            JSON_CODEC.to_string(),
10064            None,
10065        )
10066        .expect_err("fire must match scheduled timer identity");
10067        assert!(matches!(
10068            wrong_identity,
10069            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10070                if reason == "timer_identity_mismatch"
10071        ));
10072
10073        let duplicate_fire = WorkflowState::new(
10074            vec![
10075                history_event(
10076                    "TimerScheduled",
10077                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10078                ),
10079                history_event(
10080                    "TimerFired",
10081                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10082                ),
10083                history_event(
10084                    "TimerFired",
10085                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10086                ),
10087            ],
10088            "rust-workers".to_string(),
10089            JSON_CODEC.to_string(),
10090            None,
10091        )
10092        .expect_err("a durable timer cannot fire twice");
10093        assert!(matches!(
10094            duplicate_fire,
10095            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10096                if reason == "duplicate_timer_fire"
10097        ));
10098
10099        let wrong_fired_delay = WorkflowState::new(
10100            vec![
10101                history_event(
10102                    "TimerScheduled",
10103                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10104                ),
10105                history_event(
10106                    "TimerFired",
10107                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10108                ),
10109            ],
10110            "rust-workers".to_string(),
10111            JSON_CODEC.to_string(),
10112            None,
10113        )
10114        .expect_err("timer schedule and fire delays must agree");
10115        assert!(matches!(
10116            wrong_fired_delay,
10117            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10118                if reason == "timer_history_delay_mismatch"
10119        ));
10120    }
10121
10122    #[test]
10123    fn replay_rejects_activity_moved_before_recorded_timer() {
10124        let ctx = workflow_context(vec![
10125            history_event(
10126                "TimerScheduled",
10127                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10128            ),
10129            history_event(
10130                "TimerFired",
10131                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10132            ),
10133            history_event(
10134                "ActivityCompleted",
10135                json!({
10136                    "sequence": 2,
10137                    "activity_type": "after-timer",
10138                    "payload_codec": "json",
10139                    "result": {"codec": "json", "blob": "\"done\""},
10140                }),
10141            ),
10142        ]);
10143        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10144        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10145
10146        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10147            activity.as_mut().poll(&mut task_context)
10148        else {
10149            panic!("reordered durable command must be rejected");
10150        };
10151        assert_eq!(failure.reason, "recorded_command_mismatch");
10152        assert_eq!(failure.sequence, Some(1));
10153        assert_eq!(failure.expected.as_deref(), Some("timer"));
10154        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10155    }
10156
10157    #[test]
10158    fn workflow_context_emits_a_typed_named_signal_wait() {
10159        let ctx = workflow_context(Vec::new());
10160        let mut signal = Box::pin(ctx.wait_signal("finish"));
10161        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10162
10163        assert!(matches!(
10164            signal.as_mut().poll(&mut task_context),
10165            Poll::Pending
10166        ));
10167        assert_eq!(
10168            ctx.take_commands().expect("signal-wait command"),
10169            vec![json!({
10170                "type": "open_signal_wait",
10171                "signal_name": "finish",
10172            })]
10173        );
10174    }
10175
10176    #[test]
10177    fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10178        let ctx = workflow_context(vec![
10179            history_event(
10180                "ConditionWaitOpened",
10181                json!({"sequence": 1, "condition_key": "signal:finish"}),
10182            ),
10183            history_event(
10184                "ConditionWaitSatisfied",
10185                json!({"sequence": 1, "condition_key": "signal:finish"}),
10186            ),
10187            history_event(
10188                "SignalReceived",
10189                json!({"signal_name": "finish", "arguments": []}),
10190            ),
10191        ]);
10192        let mut signal = Box::pin(ctx.wait_signal("finish"));
10193        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10194
10195        assert!(matches!(
10196            signal.as_mut().poll(&mut task_context),
10197            Poll::Pending
10198        ));
10199        assert_eq!(
10200            ctx.take_commands().expect("typed signal-wait command"),
10201            vec![json!({
10202                "type": "open_signal_wait",
10203                "signal_name": "finish",
10204            })]
10205        );
10206    }
10207
10208    #[test]
10209    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10210        let signal_then_timer = vec![
10211            history_event(
10212                "SignalWaitOpened",
10213                json!({"sequence": 1, "signal_name": "go"}),
10214            ),
10215            history_event(
10216                "SignalApplied",
10217                json!({
10218                    "sequence": 1,
10219                    "signal_name": "go",
10220                    "value": {"codec": "json", "blob": "[\"now\"]"},
10221                }),
10222            ),
10223            history_event(
10224                "TimerScheduled",
10225                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10226            ),
10227            history_event(
10228                "TimerFired",
10229                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10230            ),
10231        ];
10232
10233        let ctx = workflow_context(signal_then_timer.clone());
10234        let mut signal = Box::pin(ctx.wait_signal("go"));
10235        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10236        assert!(matches!(
10237            signal.as_mut().poll(&mut task_context),
10238            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10239        ));
10240        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10241        assert!(matches!(
10242            timer.as_mut().poll(&mut task_context),
10243            Poll::Ready(Ok(()))
10244        ));
10245        ctx.ensure_history_consumed()
10246            .expect("signal and timer history consumed in order");
10247
10248        let reordered = workflow_context(signal_then_timer);
10249        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10250        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10251            timer_first.as_mut().poll(&mut task_context)
10252        else {
10253            panic!("timer cannot consume signal-wait-first history");
10254        };
10255        assert_eq!(failure.reason, "recorded_command_mismatch");
10256        assert_eq!(failure.sequence, Some(1));
10257        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10258
10259        let timer_then_signal = vec![
10260            history_event(
10261                "TimerScheduled",
10262                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10263            ),
10264            history_event(
10265                "TimerFired",
10266                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10267            ),
10268            history_event(
10269                "SignalWaitOpened",
10270                json!({"sequence": 2, "signal_name": "go"}),
10271            ),
10272            history_event(
10273                "SignalApplied",
10274                json!({
10275                    "sequence": 2,
10276                    "signal_name": "go",
10277                    "value": {"codec": "json", "blob": "[]"},
10278                }),
10279            ),
10280        ];
10281        let reordered = workflow_context(timer_then_signal);
10282        let mut signal_first = Box::pin(reordered.wait_signal("go"));
10283        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10284            signal_first.as_mut().poll(&mut task_context)
10285        else {
10286            panic!("signal wait cannot consume timer-first history");
10287        };
10288        assert_eq!(failure.reason, "recorded_command_mismatch");
10289        assert_eq!(failure.sequence, Some(1));
10290        assert_eq!(failure.expected.as_deref(), Some("timer"));
10291    }
10292
10293    #[test]
10294    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10295        let duplicate_timer = WorkflowState::new(
10296            vec![
10297                history_event(
10298                    "TimerScheduled",
10299                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10300                ),
10301                history_event(
10302                    "TimerScheduled",
10303                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10304                ),
10305            ],
10306            "rust-workers".to_string(),
10307            JSON_CODEC.to_string(),
10308            None,
10309        )
10310        .expect_err("one workflow sequence cannot schedule two timers");
10311        assert!(matches!(
10312            duplicate_timer,
10313            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10314                if reason == "timer_schedule_missing_or_duplicate"
10315        ));
10316
10317        let colliding_kinds = WorkflowState::new(
10318            vec![
10319                history_event(
10320                    "TimerScheduled",
10321                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10322                ),
10323                history_event(
10324                    "ActivityCompleted",
10325                    json!({"sequence": 1, "activity_type": "same-sequence"}),
10326                ),
10327            ],
10328            "rust-workers".to_string(),
10329            JSON_CODEC.to_string(),
10330            None,
10331        )
10332        .expect_err("one workflow sequence cannot identify two command kinds");
10333        assert!(matches!(
10334            colliding_kinds,
10335            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10336                if reason == "durable_command_sequence_collision"
10337        ));
10338
10339        let duplicate_signal_wait = WorkflowState::new(
10340            vec![
10341                history_event(
10342                    "SignalWaitOpened",
10343                    json!({"sequence": 1, "signal_name": "go"}),
10344                ),
10345                history_event(
10346                    "SignalWaitOpened",
10347                    json!({"sequence": 1, "signal_name": "go"}),
10348                ),
10349            ],
10350            "rust-workers".to_string(),
10351            JSON_CODEC.to_string(),
10352            None,
10353        )
10354        .expect_err("one workflow sequence cannot open two signal waits");
10355        assert!(matches!(
10356            duplicate_signal_wait,
10357            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10358                if reason == "signal_wait_open_missing_or_duplicate"
10359        ));
10360    }
10361
10362    #[test]
10363    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10364        let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10365            .expect("side-effect result");
10366        let ctx = workflow_context(vec![history_event(
10367            "SideEffectRecorded",
10368            json!({"sequence": 99, "result": result}),
10369        )]);
10370
10371        let replayed: Value = ctx
10372            .side_effect(|| panic!("recorded side effect must not run"))
10373            .expect("positive global workflow sequence is valid");
10374        assert_eq!(replayed, json!({"captured": true}));
10375        ctx.ensure_history_consumed().expect("history consumed");
10376    }
10377
10378    #[test]
10379    fn workflow_history_rejects_zero_and_descending_command_sequences() {
10380        let result =
10381            encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10382        let zero = WorkflowState::new(
10383            vec![history_event(
10384                "SideEffectRecorded",
10385                json!({"sequence": 0, "result": result.clone()}),
10386            )],
10387            "rust-workers".to_string(),
10388            JSON_CODEC.to_string(),
10389            None,
10390        )
10391        .expect_err("durable command sequences must be positive");
10392        assert!(matches!(
10393            zero,
10394            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10395                if reason == "durable_command_sequence_invalid"
10396        ));
10397
10398        let descending = WorkflowState::new(
10399            vec![
10400                history_event(
10401                    "SideEffectRecorded",
10402                    json!({"sequence": 3, "result": result}),
10403                ),
10404                history_event(
10405                    "VersionMarkerRecorded",
10406                    json!({
10407                        "sequence": 2,
10408                        "change_id": "descending-marker",
10409                        "version": 1,
10410                        "min_supported": 1,
10411                        "max_supported": 1,
10412                    }),
10413                ),
10414            ],
10415            "rust-workers".to_string(),
10416            JSON_CODEC.to_string(),
10417            None,
10418        )
10419        .expect_err("new durable commands must remain strictly ordered");
10420        let Error::NonDeterministicReplay(failure) = descending else {
10421            panic!("expected typed replay failure");
10422        };
10423        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10424        assert_eq!(failure.sequence, Some(2));
10425        assert_eq!(
10426            failure.expected.as_deref(),
10427            Some("workflow sequence greater than 3")
10428        );
10429        assert_eq!(failure.actual.as_deref(), Some("2"));
10430    }
10431
10432    #[test]
10433    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10434        fn worker() -> Worker {
10435            let client = Client::new("http://127.0.0.1:8080").expect("client");
10436            let mut worker = Worker::new(client, "rust-workers");
10437            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10438                ctx.wait_signal("finish").await?;
10439                let marker: String =
10440                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10441                assert_eq!(marker, "after-finish");
10442                Ok(json!("finished"))
10443            });
10444            worker
10445        }
10446
10447        let marker =
10448            encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10449        let task = workflow_task(
10450            "rust.finish-after-gaps",
10451            vec![
10452                history_event(
10453                    "SignalWaitOpened",
10454                    json!({"sequence": 1, "signal_name": "finish"}),
10455                ),
10456                history_event(
10457                    "SignalReceived",
10458                    json!({
10459                        "signal_id": "increment-3",
10460                        "signal_name": "increment",
10461                        "workflow_sequence": 2,
10462                        "payload_codec": "json",
10463                        "arguments": {"codec": "json", "blob": "[3]"},
10464                    }),
10465                ),
10466                history_event(
10467                    "SignalReceived",
10468                    json!({
10469                        "signal_id": "increment-5",
10470                        "signal_name": "increment",
10471                        "workflow_sequence": 3,
10472                        "payload_codec": "json",
10473                        "arguments": {"codec": "json", "blob": "[5]"},
10474                    }),
10475                ),
10476                history_event(
10477                    "SignalReceived",
10478                    json!({
10479                        "signal_id": "finish",
10480                        "signal_name": "finish",
10481                        "workflow_sequence": 4,
10482                        "payload_codec": "json",
10483                        "arguments": {"codec": "json", "blob": "[]"},
10484                    }),
10485                ),
10486                history_event(
10487                    "SignalApplied",
10488                    json!({
10489                        "sequence": 1,
10490                        "signal_id": "finish",
10491                        "signal_name": "finish",
10492                        "payload_codec": "json",
10493                        "value": {"codec": "json", "blob": "[]"},
10494                    }),
10495                ),
10496                history_event(
10497                    "SideEffectRecorded",
10498                    json!({"sequence": 5, "result": marker}),
10499                ),
10500            ],
10501            JSON_CODEC,
10502        );
10503
10504        for _original_or_cold_worker in 0..2 {
10505            let commands = worker()
10506                .execute_workflow_task(task.clone())
10507                .expect("signal gaps preserve deterministic replay");
10508            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10509            assert_eq!(commands[0]["type"], "complete_workflow");
10510            assert_eq!(
10511                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10512                json!("finished")
10513            );
10514        }
10515    }
10516
10517    #[test]
10518    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10519        let ctx = workflow_context(Vec::new());
10520        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10521        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10522        assert!(matches!(
10523            sleep.as_mut().poll(&mut task_context),
10524            Poll::Ready(Err(Error::TimerDurationOverflow))
10525        ));
10526        assert!(ctx.take_commands().expect("commands").is_empty());
10527    }
10528
10529    #[test]
10530    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10531        let client = Client::new("http://127.0.0.1:8080").expect("client");
10532        let mut worker = Worker::new(client, "rust-workers");
10533        worker.register_workflow("rust.timer", |ctx, _input| async move {
10534            ctx.sleep(Duration::from_secs(5)).await?;
10535            ctx.activity("after-timer", json!([])).await
10536        });
10537
10538        let task = |history_events| WorkflowTask {
10539            task_id: "wft-rust-timer-1".to_string(),
10540            workflow_id: Some("wf-rust-timer".to_string()),
10541            run_id: Some("run-rust-timer".to_string()),
10542            workflow_type: "rust.timer".to_string(),
10543            payload_codec: JSON_CODEC.to_string(),
10544            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10545            history_events,
10546            total_history_events: None,
10547            history_size_bytes: None,
10548            continue_as_new_recommended: None,
10549            history_budget_pressure: None,
10550            next_history_page_token: None,
10551            workflow_task_attempt: 1,
10552            workflow_signal_id: None,
10553            signal_name: None,
10554            signal_arguments: None,
10555            workflow_update_id: None,
10556            update_name: None,
10557            lease_owner: Some("rust-worker".to_string()),
10558        };
10559
10560        let initial = worker
10561            .execute_workflow_task(task(Vec::new()))
10562            .expect("initial timer task");
10563        assert_eq!(
10564            initial,
10565            vec![json!({"type": "start_timer", "delay_seconds": 5})]
10566        );
10567
10568        let replayed = worker
10569            .execute_workflow_task(task(vec![
10570                history_event(
10571                    "TimerScheduled",
10572                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10573                ),
10574                history_event(
10575                    "TimerFired",
10576                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10577                ),
10578                history_event(
10579                    "ActivityCompleted",
10580                    json!({
10581                        "sequence": 2,
10582                        "activity_type": "after-timer",
10583                        "payload_codec": "json",
10584                        "result": {"codec": "json", "blob": "\"done\""},
10585                    }),
10586                ),
10587            ]))
10588            .expect("replayed workflow task");
10589        assert_eq!(replayed.len(), 1);
10590        assert_eq!(replayed[0]["type"], "complete_workflow");
10591        assert_eq!(
10592            decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10593            json!("done")
10594        );
10595    }
10596
10597    #[test]
10598    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10599        let client = Client::new("http://127.0.0.1:8080").expect("client");
10600        let mut worker = Worker::new(client, "rust-workers");
10601        worker.register_workflow("rust.continue", |ctx, _input| async move {
10602            ctx.continue_as_new_with_options(
10603                ContinueAsNewOptions::new()
10604                    .workflow_type("rust.next")
10605                    .task_queue("next-workers"),
10606                json!([2, {"cursor": "next"}]),
10607            )
10608        });
10609
10610        let commands = worker
10611            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10612            .expect("continue-as-new command");
10613
10614        assert_eq!(commands.len(), 1);
10615        assert_eq!(commands[0]["type"], "continue_as_new");
10616        assert_eq!(commands[0]["workflow_type"], "rust.next");
10617        assert_eq!(commands[0]["queue"], "next-workers");
10618        assert_eq!(
10619            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10620                .expect("continue-as-new arguments"),
10621            json!([2, {"cursor": "next"}])
10622        );
10623    }
10624
10625    #[test]
10626    fn continue_as_new_preserves_typed_arguments() {
10627        let client = Client::new("http://127.0.0.1:8080").expect("client");
10628        let mut worker = Worker::new(client, "rust-workers");
10629        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10630            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10631            unreachable!("continue-as-new returns a control-flow error")
10632        });
10633
10634        let commands = worker
10635            .execute_workflow_task(workflow_task(
10636                "rust.typed-continue",
10637                Vec::new(),
10638                DEFAULT_CODEC,
10639            ))
10640            .expect("typed continue-as-new command");
10641
10642        assert_eq!(commands[0]["type"], "continue_as_new");
10643        assert_eq!(
10644            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10645                .expect("typed continue arguments"),
10646            AvroValue::Array(vec![typed_fidelity_probe()])
10647        );
10648    }
10649
10650    #[test]
10651    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10652        let client = Client::new("http://127.0.0.1:8080").expect("client");
10653        let mut worker = Worker::new(client, "rust-workers");
10654        worker.register_workflow("rust.continue", |ctx, _input| async move {
10655            ctx.continue_as_new(json!([2]))
10656        });
10657        let task = workflow_task(
10658            "rust.continue",
10659            vec![history_event(
10660                "WorkflowContinuedAsNew",
10661                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10662            )],
10663            JSON_CODEC,
10664        );
10665
10666        for _worker_restart_or_redelivery in 0..2 {
10667            let commands = worker
10668                .execute_workflow_task(task.clone())
10669                .expect("recorded transition replays");
10670            assert!(
10671                commands.is_empty(),
10672                "replay must not emit another successor"
10673            );
10674        }
10675    }
10676
10677    #[test]
10678    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10679        let ctx = workflow_context(Vec::new());
10680        let error = ctx
10681            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
10682            .expect_err("blank queue must be rejected");
10683
10684        let Error::InvalidContinueAsNewOptions(error) = error else {
10685            panic!("expected typed continue-as-new validation error");
10686        };
10687        assert_eq!(error.field, "task_queue");
10688        assert!(ctx.take_commands().expect("commands").is_empty());
10689    }
10690
10691    #[test]
10692    fn workflow_context_exposes_server_history_budget() {
10693        let client = Client::new("http://127.0.0.1:8080").expect("client");
10694        let mut worker = Worker::new(client, "rust-workers");
10695        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10696            let budget = ctx.history_budget()?;
10697            Ok(json!({
10698                "events": budget.event_count,
10699                "bytes": budget.size_bytes,
10700                "recommended": budget.continue_as_new_recommended,
10701                "pressure": budget.pressure,
10702            }))
10703        });
10704        let task: WorkflowTask = serde_json::from_value(json!({
10705            "task_id": "task-history-budget",
10706            "workflow_type": "rust.history-budget",
10707            "payload_codec": JSON_CODEC,
10708            "history_events": [],
10709            "total_history_events": 480,
10710            "history_size_bytes": 1_048_576,
10711            "continue_as_new_recommended": true,
10712            "history_budget_pressure": "continue_as_new_recommended",
10713        }))
10714        .expect("published workflow task");
10715
10716        let commands = worker
10717            .execute_workflow_task(task)
10718            .expect("history-budget workflow");
10719        let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10720        assert_eq!(result["events"], 480);
10721        assert_eq!(result["bytes"], 1_048_576);
10722        assert_eq!(result["recommended"], true);
10723        assert_eq!(result["pressure"], "continue_as_new_recommended");
10724    }
10725
10726    #[test]
10727    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10728        let client = Client::new("http://127.0.0.1:8080").expect("client");
10729        let mut worker = Worker::new(client, "rust-workers");
10730        worker.register_workflow("rust.failing", |_ctx, _input| async move {
10731            Err(Error::Codec("rust_conformance_failure".to_string()))
10732        });
10733        let task = WorkflowTask {
10734            task_id: "wft-rust-failing-1".to_string(),
10735            workflow_id: Some("wf-rust-failing".to_string()),
10736            run_id: Some("run-rust-failing".to_string()),
10737            workflow_type: "rust.failing".to_string(),
10738            payload_codec: JSON_CODEC.to_string(),
10739            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10740            history_events: Vec::new(),
10741            total_history_events: Some(0),
10742            history_size_bytes: None,
10743            continue_as_new_recommended: None,
10744            history_budget_pressure: None,
10745            next_history_page_token: None,
10746            workflow_task_attempt: 1,
10747            workflow_signal_id: None,
10748            signal_name: None,
10749            signal_arguments: None,
10750            workflow_update_id: None,
10751            update_name: None,
10752            lease_owner: Some("rust-worker".to_string()),
10753        };
10754
10755        let commands = worker
10756            .execute_workflow_task(task)
10757            .expect("handler failure becomes a workflow command");
10758
10759        assert_eq!(commands.len(), 1);
10760        assert_eq!(commands[0]["type"], "fail_workflow");
10761        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10762        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10763        assert_eq!(commands[0]["non_retryable"], false);
10764        assert_eq!(
10765            commands[0]["message"],
10766            "codec error: rust_conformance_failure"
10767        );
10768        assert_eq!(
10769            commands[0]["exception"]["message"],
10770            "codec error: rust_conformance_failure"
10771        );
10772    }
10773
10774    #[test]
10775    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10776        let client = Client::new("http://127.0.0.1:8080").expect("client");
10777        let mut worker = Worker::new(client, "rust-workers");
10778        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10779            let _: String = ctx.side_effect(|| "captured".to_string())?;
10780            Err(Error::WorkerLoop("application failure".to_string()))
10781        });
10782
10783        let commands = worker
10784            .execute_workflow_task(workflow_task(
10785                "rust.failing-after-side-effect",
10786                Vec::new(),
10787                JSON_CODEC,
10788            ))
10789            .expect("ordinary failure remains a workflow decision");
10790
10791        assert_eq!(commands.len(), 2);
10792        assert_eq!(commands[0]["type"], "record_side_effect");
10793        assert_eq!(commands[1]["type"], "fail_workflow");
10794    }
10795
10796    #[test]
10797    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10798        let client = Client::new("http://127.0.0.1:8080").expect("client");
10799        let mut worker = Worker::new(client, "rust-workers");
10800        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10801            Err(Error::WorkerLoop("application failure".to_string()))
10802        });
10803        let result =
10804            encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10805
10806        let error = worker
10807            .execute_workflow_task(workflow_task(
10808                "rust.removed-side-effect",
10809                vec![history_event(
10810                    "SideEffectRecorded",
10811                    json!({"sequence": 1, "result": result}),
10812                )],
10813                JSON_CODEC,
10814            ))
10815            .expect_err("removed committed history must not become fail_workflow");
10816
10817        let Error::NonDeterministicReplay(failure) = error else {
10818            panic!("expected typed replay failure");
10819        };
10820        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10821        assert_eq!(failure.sequence, Some(1));
10822        assert_eq!(failure.expected.as_deref(), Some("side effect"));
10823    }
10824
10825    #[test]
10826    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10827        let client = Client::new("http://127.0.0.1:8080").expect("client");
10828        let mut worker = Worker::new(client, "rust-workers");
10829        worker.register_workflow(
10830            "rust.side-effect-before-marker-error",
10831            |ctx, _input| async move {
10832                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10833                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10834                ctx.get_version("restart-safe", 2, 2)?;
10835                Ok(Value::Null)
10836            },
10837        );
10838
10839        let error = worker
10840            .execute_workflow_task(workflow_task(
10841                "rust.side-effect-before-marker-error",
10842                vec![history_event(
10843                    "VersionMarkerRecorded",
10844                    json!({
10845                        "sequence": 1,
10846                        "change_id": "restart-safe",
10847                        "version": 1,
10848                        "min_supported": 1,
10849                        "max_supported": 1,
10850                    }),
10851                )],
10852                JSON_CODEC,
10853            ))
10854            .expect_err("replay error must return no queued workflow commands");
10855
10856        let Error::NonDeterministicReplay(failure) = error else {
10857            panic!("expected typed replay failure");
10858        };
10859        assert_eq!(failure.reason, "version_marker_incompatible_range");
10860        assert_eq!(failure.sequence, Some(1));
10861    }
10862
10863    #[test]
10864    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10865        let client = Client::new("http://127.0.0.1:8080").expect("client");
10866        let mut worker = Worker::new(client, "rust-workers");
10867        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10868            ctx.sleep(Duration::from_secs(5)).await?;
10869            Ok(json!({"status": "timer fired"}))
10870        });
10871
10872        let task = WorkflowTask {
10873            task_id: "wft-rust-timer-pending".to_string(),
10874            workflow_id: Some("wf-rust-timer".to_string()),
10875            run_id: Some("run-rust-timer".to_string()),
10876            workflow_type: "rust.timer.pending".to_string(),
10877            payload_codec: JSON_CODEC.to_string(),
10878            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10879            history_events: vec![history_event(
10880                "TimerScheduled",
10881                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10882            )],
10883            total_history_events: Some(1),
10884            history_size_bytes: None,
10885            continue_as_new_recommended: None,
10886            history_budget_pressure: None,
10887            next_history_page_token: None,
10888            workflow_task_attempt: 1,
10889            workflow_signal_id: None,
10890            signal_name: None,
10891            signal_arguments: None,
10892            workflow_update_id: None,
10893            update_name: None,
10894            lease_owner: Some("rust-worker".to_string()),
10895        };
10896
10897        for _redelivery_or_restart in 0..2 {
10898            let commands = worker
10899                .execute_workflow_task(task.clone())
10900                .expect("recorded timer remains pending");
10901            assert!(
10902                commands.is_empty(),
10903                "recorded timer must not be rescheduled"
10904            );
10905        }
10906    }
10907
10908    #[test]
10909    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10910        let client = Client::new("http://127.0.0.1:8080").expect("client");
10911        let mut worker = Worker::new(client, "rust-workers");
10912        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10913            Ok(json!({"status": "completed"}))
10914        });
10915        let task = WorkflowTask {
10916            task_id: "wft-rust-timer-removed".to_string(),
10917            workflow_id: Some("wf-rust-timer".to_string()),
10918            run_id: Some("run-rust-timer".to_string()),
10919            workflow_type: "rust.timer.removed".to_string(),
10920            payload_codec: JSON_CODEC.to_string(),
10921            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10922            history_events: vec![
10923                history_event(
10924                    "TimerScheduled",
10925                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10926                ),
10927                history_event(
10928                    "TimerFired",
10929                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10930                ),
10931            ],
10932            total_history_events: Some(2),
10933            history_size_bytes: None,
10934            continue_as_new_recommended: None,
10935            history_budget_pressure: None,
10936            next_history_page_token: None,
10937            workflow_task_attempt: 1,
10938            workflow_signal_id: None,
10939            signal_name: None,
10940            signal_arguments: None,
10941            workflow_update_id: None,
10942            update_name: None,
10943            lease_owner: Some("rust-worker".to_string()),
10944        };
10945
10946        let Error::NonDeterministicReplay(failure) = worker
10947            .execute_workflow_task(task)
10948            .expect_err("removed timer must fail replay")
10949        else {
10950            panic!("expected typed replay failure");
10951        };
10952        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10953        assert_eq!(failure.sequence, Some(1));
10954    }
10955
10956    #[test]
10957    fn workflow_context_emits_explicit_child_workflow_contract() {
10958        let ctx = WorkflowContext {
10959            state: Arc::new(Mutex::new(
10960                WorkflowState::new_with_identity(
10961                    Vec::new(),
10962                    Some("wf-parent".to_string()),
10963                    Some("run-parent".to_string()),
10964                    "parent-workers".to_string(),
10965                    JSON_CODEC.to_string(),
10966                    None,
10967                )
10968                .expect("workflow state"),
10969            )),
10970        };
10971        let options = ChildWorkflowOptions::new("python-workers")
10972            .parent_close_policy(ParentClosePolicy::RequestCancel)
10973            .retry_policy(ChildWorkflowRetryPolicy {
10974                max_attempts: Some(3),
10975                backoff_seconds: vec![1, 5],
10976                non_retryable_error_types: vec!["ValidationError".to_string()],
10977            })
10978            .execution_timeout_seconds(600)
10979            .run_timeout_seconds(120);
10980        let mut call = Box::pin(ctx.start_child_workflow(
10981            "python.fulfil-order",
10982            options,
10983            json!([{"order_id": "order-42"}]),
10984        ));
10985        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10986
10987        assert!(matches!(
10988            call.as_mut().poll(&mut task_context),
10989            Poll::Pending
10990        ));
10991        let commands = ctx.take_commands().expect("commands");
10992        assert_eq!(commands.len(), 1);
10993        let command = &commands[0];
10994        assert_eq!(command["type"], "start_child_workflow");
10995        assert_eq!(command["workflow_type"], "python.fulfil-order");
10996        assert_eq!(command["queue"], "python-workers");
10997        assert_eq!(command["parent_close_policy"], "request_cancel");
10998        assert_eq!(command["retry_policy"]["max_attempts"], 3);
10999        assert_eq!(command["execution_timeout_seconds"], 600);
11000        assert_eq!(command["run_timeout_seconds"], 120);
11001        assert_eq!(
11002            decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
11003            json!([{"order_id": "order-42"}])
11004        );
11005    }
11006
11007    fn child_parent_worker() -> Worker {
11008        let client = Client::new("http://127.0.0.1:8080").expect("client");
11009        let mut worker = Worker::new(client, "rust-parent-workers");
11010        worker.register_workflow("rust.parent", |ctx, _input| async move {
11011            let child = ctx
11012                .start_child_workflow(
11013                    "python.child",
11014                    ChildWorkflowOptions::new("python-child-workers")
11015                        .parent_close_policy(ParentClosePolicy::Terminate),
11016                    json!([{"codec_probe": [1, true, "rust"]}]),
11017                )
11018                .await?;
11019            Ok(json!({
11020                "parent_workflow_id": child.parent.workflow_id,
11021                "parent_run_id": child.parent.run_id,
11022                "child_workflow_id": child.child.workflow_id,
11023                "child_run_id": child.child.run_id,
11024                "child_workflow_type": child.child_workflow_type,
11025                "result": child.result,
11026            }))
11027        });
11028        worker
11029    }
11030
11031    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11032        WorkflowTask {
11033            task_id: "wft-child-parent".to_string(),
11034            workflow_id: Some("wf-parent".to_string()),
11035            run_id: Some("run-parent".to_string()),
11036            workflow_type: "rust.parent".to_string(),
11037            payload_codec: JSON_CODEC.to_string(),
11038            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
11039            history_events: vec![
11040                HistoryEvent {
11041                    event_type: "ChildWorkflowScheduled".to_string(),
11042                    payload: json!({
11043                        "sequence": 1,
11044                        "child_call_id": "call-child",
11045                        "child_workflow_instance_id": "wf-child",
11046                        "child_workflow_run_id": "run-child",
11047                        "child_workflow_type": "python.child",
11048                    }),
11049                    raw: HashMap::new(),
11050                },
11051                HistoryEvent {
11052                    event_type: event_type.to_string(),
11053                    payload,
11054                    raw: HashMap::new(),
11055                },
11056            ],
11057            total_history_events: Some(2),
11058            history_size_bytes: None,
11059            continue_as_new_recommended: None,
11060            history_budget_pressure: None,
11061            next_history_page_token: None,
11062            workflow_task_attempt: 1,
11063            workflow_signal_id: None,
11064            signal_name: None,
11065            signal_arguments: None,
11066            workflow_update_id: None,
11067            update_name: None,
11068            lease_owner: Some("rust-worker".to_string()),
11069        }
11070    }
11071
11072    #[test]
11073    fn committed_child_result_replays_without_starting_a_duplicate() {
11074        let worker = child_parent_worker();
11075        let task = child_parent_task(
11076            "ChildRunCompleted",
11077            json!({
11078                "sequence": 1,
11079                "child_call_id": "call-child",
11080                "child_workflow_instance_id": "wf-child",
11081                "child_workflow_run_id": "run-child",
11082                "child_workflow_type": "python.child",
11083                "payload_codec": "json",
11084                "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
11085            }),
11086        );
11087
11088        for _restart in 0..2 {
11089            let commands = worker
11090                .execute_workflow_task(task.clone())
11091                .expect("replayed parent task");
11092            assert_eq!(commands.len(), 1);
11093            assert_eq!(commands[0]["type"], "complete_workflow");
11094            assert!(!commands
11095                .iter()
11096                .any(|command| command["type"] == "start_child_workflow"));
11097            let output =
11098                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11099            assert_eq!(output["parent_workflow_id"], "wf-parent");
11100            assert_eq!(output["parent_run_id"], "run-parent");
11101            assert_eq!(output["child_workflow_id"], "wf-child");
11102            assert_eq!(output["child_run_id"], "run-child");
11103            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11104        }
11105    }
11106
11107    #[test]
11108    fn typed_child_arguments_and_results_survive_replay() {
11109        let client = Client::new("http://127.0.0.1:8080").expect("client");
11110        let mut worker = Worker::new(client, "rust-parent-workers");
11111        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11112            let child = ctx
11113                .start_child_workflow_avro_value(
11114                    "python.typed-child",
11115                    ChildWorkflowOptions::new("python-workers"),
11116                    AvroValue::Array(vec![typed_fidelity_probe()]),
11117                )
11118                .await?;
11119            Ok(child.result)
11120        });
11121
11122        let initial = worker
11123            .execute_workflow_task(workflow_task(
11124                "rust.typed-parent",
11125                Vec::new(),
11126                DEFAULT_CODEC,
11127            ))
11128            .expect("typed child start");
11129        assert_eq!(initial[0]["type"], "start_child_workflow");
11130        assert_eq!(
11131            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11132                .expect("typed child arguments"),
11133            AvroValue::Array(vec![typed_fidelity_probe()])
11134        );
11135
11136        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11137            .expect("typed child result");
11138        let task = workflow_task(
11139            "rust.typed-parent",
11140            vec![
11141                history_event(
11142                    "ChildWorkflowScheduled",
11143                    json!({
11144                        "sequence": 1,
11145                        "child_call_id": "call-typed",
11146                        "child_workflow_instance_id": "wf-child",
11147                        "child_workflow_run_id": "run-child",
11148                        "child_workflow_type": "python.typed-child",
11149                    }),
11150                ),
11151                history_event(
11152                    "ChildRunCompleted",
11153                    json!({
11154                        "sequence": 1,
11155                        "child_call_id": "call-typed",
11156                        "child_workflow_instance_id": "wf-child",
11157                        "child_workflow_run_id": "run-child",
11158                        "child_workflow_type": "python.typed-child",
11159                        "payload_codec": DEFAULT_CODEC,
11160                        "result": result,
11161                    }),
11162                ),
11163            ],
11164            DEFAULT_CODEC,
11165        );
11166
11167        let commands = worker
11168            .execute_workflow_task(task)
11169            .expect("typed child replay");
11170        assert_eq!(commands[0]["type"], "complete_workflow");
11171        assert_eq!(
11172            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11173                .expect("typed parent result"),
11174            typed_fidelity_probe()
11175        );
11176    }
11177
11178    #[test]
11179    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11180        let worker = child_parent_worker();
11181        let mut task = child_parent_task("unused", Value::Null);
11182        task.history_events.truncate(1);
11183        task.total_history_events = Some(1);
11184
11185        for _redelivery_or_restart in 0..2 {
11186            let commands = worker
11187                .execute_workflow_task(task.clone())
11188                .expect("recorded child remains pending");
11189            assert!(
11190                commands.is_empty(),
11191                "recorded pending child must not be started again"
11192            );
11193        }
11194    }
11195
11196    #[test]
11197    fn child_cancellation_becomes_stable_parent_failure_command() {
11198        let worker = child_parent_worker();
11199        let task = child_parent_task(
11200            "ChildRunCancelled",
11201            json!({
11202                "sequence": 1,
11203                "child_workflow_instance_id": "wf-child",
11204                "child_workflow_run_id": "run-child",
11205                "child_workflow_type": "python.child",
11206                "failure_id": "failure-child",
11207                "failure_category": "cancelled",
11208                "message": "cancelled by parent-close policy",
11209            }),
11210        );
11211
11212        let commands = worker
11213            .execute_workflow_task(task)
11214            .expect("parent settlement");
11215        assert_eq!(commands.len(), 1);
11216        assert_eq!(commands[0]["type"], "fail_workflow");
11217        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11218        assert_eq!(
11219            commands[0]["exception"]["properties"]["reason"],
11220            "cancelled"
11221        );
11222        assert_eq!(
11223            commands[0]["exception"]["properties"]["child_workflow_run_id"],
11224            "run-child"
11225        );
11226    }
11227
11228    #[test]
11229    fn workflow_can_handle_typed_child_failure() {
11230        let client = Client::new("http://127.0.0.1:8080").expect("client");
11231        let mut worker = Worker::new(client, "rust-parent-workers");
11232        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11233            match ctx
11234                .start_child_workflow(
11235                    "python.child",
11236                    ChildWorkflowOptions::new("python-child-workers"),
11237                    json!([]),
11238                )
11239                .await
11240            {
11241                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11242                    "reason": failure.reason,
11243                    "failure_id": failure.failure_id,
11244                    "exception_class": failure.exception_class,
11245                    "child_run_id": failure.child_workflow_run_id,
11246                })),
11247                Err(error) => Err(error),
11248                Ok(_) => Err(Error::WorkerLoop(
11249                    "child unexpectedly succeeded".to_string(),
11250                )),
11251            }
11252        });
11253        let mut task = child_parent_task(
11254            "ChildRunFailed",
11255            json!({
11256                "sequence": 1,
11257                "child_workflow_instance_id": "wf-child",
11258                "child_workflow_run_id": "run-child",
11259                "child_workflow_type": "python.child",
11260                "failure_id": "failure-child",
11261                "failure_category": "child_workflow",
11262                "message": "payment rejected",
11263                "exception": {
11264                    "type": "PaymentRejected",
11265                    "class": "payments.PaymentRejected",
11266                    "message": "payment rejected"
11267                }
11268            }),
11269        );
11270        task.workflow_type = "rust.handled-parent".to_string();
11271
11272        let commands = worker.execute_workflow_task(task).expect("handled failure");
11273        assert_eq!(commands[0]["type"], "complete_workflow");
11274        let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11275        assert_eq!(output["reason"], "child_workflow");
11276        assert_eq!(output["failure_id"], "failure-child");
11277        assert_eq!(output["exception_class"], "payments.PaymentRejected");
11278        assert_eq!(output["child_run_id"], "run-child");
11279    }
11280
11281    #[test]
11282    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11283        let client = Client::new("http://127.0.0.1:8080").expect("client");
11284        let mut worker = Worker::new(client, "rust-workers");
11285
11286        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11287            let signal = ctx.wait_signal("start").await?;
11288            let name = signal
11289                .first()
11290                .and_then(|value| value.as_str())
11291                .unwrap_or("world");
11292            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11293            Ok(json!({
11294                "greeting": greeting,
11295                "language": "rust"
11296            }))
11297        });
11298
11299        let signal_arguments =
11300            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11301        let task = WorkflowTask {
11302            task_id: "wft-rust-signal-1".to_string(),
11303            workflow_id: Some("wf-rust-hello".to_string()),
11304            run_id: Some("run-rust-hello".to_string()),
11305            workflow_type: "rust.hello_workflow".to_string(),
11306            payload_codec: DEFAULT_CODEC.to_string(),
11307            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11308            history_events: vec![HistoryEvent {
11309                event_type: "SignalReceived".to_string(),
11310                payload: json!({
11311                    "signal_id": "sig-rust-1",
11312                    "signal_name": "start"
11313                }),
11314                raw: HashMap::new(),
11315            }],
11316            total_history_events: Some(1),
11317            history_size_bytes: None,
11318            continue_as_new_recommended: None,
11319            history_budget_pressure: None,
11320            next_history_page_token: None,
11321            workflow_task_attempt: 1,
11322            workflow_signal_id: Some("sig-rust-1".to_string()),
11323            signal_name: Some("start".to_string()),
11324            signal_arguments: Some(signal_arguments),
11325            workflow_update_id: None,
11326            update_name: None,
11327            lease_owner: Some("rust-worker".to_string()),
11328        };
11329
11330        let commands = worker.execute_workflow_task(task).expect("workflow task");
11331
11332        assert_eq!(commands.len(), 1);
11333        assert_eq!(commands[0]["type"], "schedule_activity");
11334        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11335        assert_eq!(
11336            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11337            json!(["Rust"])
11338        );
11339    }
11340
11341    #[test]
11342    fn workflow_task_appends_paginated_history_events() {
11343        let mut task = WorkflowTask {
11344            task_id: "wft-rust-pages-1".to_string(),
11345            workflow_id: Some("wf-rust-pages".to_string()),
11346            run_id: Some("run-rust-pages".to_string()),
11347            workflow_type: "rust.hello_workflow".to_string(),
11348            payload_codec: DEFAULT_CODEC.to_string(),
11349            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11350            history_events: vec![HistoryEvent {
11351                event_type: "WorkflowStarted".to_string(),
11352                payload: json!({}),
11353                raw: HashMap::new(),
11354            }],
11355            total_history_events: Some(3),
11356            history_size_bytes: None,
11357            continue_as_new_recommended: None,
11358            history_budget_pressure: None,
11359            next_history_page_token: Some("MQ==".to_string()),
11360            workflow_task_attempt: 1,
11361            workflow_signal_id: None,
11362            signal_name: None,
11363            signal_arguments: None,
11364            workflow_update_id: None,
11365            update_name: None,
11366            lease_owner: Some("rust-worker".to_string()),
11367        };
11368
11369        task.append_history_page(WorkflowTaskHistoryPage {
11370            history_events: vec![
11371                HistoryEvent {
11372                    event_type: "SignalReceived".to_string(),
11373                    payload: json!({
11374                        "signal_id": "sig-rust-1",
11375                        "signal_name": "start",
11376                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11377                            .expect("signal arguments")
11378                    }),
11379                    raw: HashMap::new(),
11380                },
11381                HistoryEvent {
11382                    event_type: "MarkerRecorded".to_string(),
11383                    payload: json!({"sequence": 3}),
11384                    raw: HashMap::new(),
11385                },
11386            ],
11387            total_history_events: Some(3),
11388            next_history_page_token: None,
11389        });
11390
11391        assert_eq!(task.history_events.len(), 3);
11392        assert_eq!(task.total_history_events, Some(3));
11393        assert_eq!(task.next_history_page_token, None);
11394
11395        let signal = task
11396            .history_events
11397            .iter()
11398            .find(|event| event.event_type == "SignalReceived")
11399            .expect("signal event");
11400        assert_eq!(
11401            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11402            vec![AvroValue::String("Rust".to_string())]
11403        );
11404    }
11405
11406    #[tokio::test]
11407    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11408        let client = Client::new("http://127.0.0.1:8080").expect("client");
11409        let mut worker = Worker::new(client, "rust-workers");
11410        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11411        worker.register_query("counter", "current", |ctx, _args| async move {
11412            let mut count = 0_i64;
11413            for signal in ctx.signal_events() {
11414                let value = signal
11415                    .arguments
11416                    .first()
11417                    .and_then(Value::as_i64)
11418                    .unwrap_or_default();
11419                match signal.name.as_str() {
11420                    "increment" => count += value,
11421                    "set" => count = value,
11422                    _ => {}
11423                }
11424            }
11425            Ok(json!(count))
11426        });
11427
11428        let task = QueryTask {
11429            query_task_id: "query-rust-counter".to_string(),
11430            query_task_attempt: 1,
11431            lease_owner: Some("rust-worker".to_string()),
11432            workflow_id: Some("counter-1".to_string()),
11433            run_id: Some("run-counter-1".to_string()),
11434            workflow_type: "counter".to_string(),
11435            query_name: "current".to_string(),
11436            payload_codec: DEFAULT_CODEC.to_string(),
11437            workflow_arguments: Some(
11438                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11439            ),
11440            query_arguments: Some(
11441                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11442            ),
11443            history_events: vec![
11444                HistoryEvent {
11445                    event_type: "SignalReceived".to_string(),
11446                    payload: json!({
11447                        "signal_id": "php-signal-1",
11448                        "signal_name": "increment",
11449                        "workflow_sequence": 1,
11450                        "payload_codec": DEFAULT_CODEC,
11451                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11452                    }),
11453                    raw: HashMap::new(),
11454                },
11455                HistoryEvent {
11456                    event_type: "SignalReceived".to_string(),
11457                    payload: json!({
11458                        "signal_id": "python-signal-2",
11459                        "signal_name": "increment",
11460                        "workflow_sequence": 2,
11461                        "payload_codec": JSON_CODEC,
11462                        "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11463                    }),
11464                    raw: HashMap::new(),
11465                },
11466                HistoryEvent {
11467                    event_type: "SignalReceived".to_string(),
11468                    payload: json!({
11469                        "signal_id": "rust-signal-3",
11470                        "signal_name": "set",
11471                        "workflow_sequence": 3,
11472                        "payload_codec": DEFAULT_CODEC,
11473                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11474                    }),
11475                    raw: HashMap::new(),
11476                },
11477            ],
11478            history_export: None,
11479            run_status: Some("completed".to_string()),
11480        };
11481
11482        let result = worker.execute_query_task(task).await.expect("query result");
11483        assert_eq!(result.into_json().expect("query projection"), json!(0));
11484    }
11485
11486    #[tokio::test]
11487    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11488        let worker = replay_counter_worker();
11489        let running_history = json!([
11490            {
11491                "type": "ActivityCompleted",
11492                "payload": {
11493                    "sequence": 1,
11494                    "activity_type": "load-counter",
11495                    "payload_codec": "json",
11496                    "result": {"codec": "json", "blob": "\"loaded\""}
11497                }
11498            },
11499            {
11500                "type": "SignalWaitOpened",
11501                "payload": {
11502                    "sequence": 3,
11503                    "signal_name": "increment"
11504                }
11505            },
11506            {
11507                "type": "SignalReceived",
11508                "payload": {
11509                    "signal_id": "signal-3",
11510                    "signal_name": "increment",
11511                    "workflow_sequence": 2,
11512                    "payload_codec": "json",
11513                    "arguments": {"codec": "json", "blob": "[3]"}
11514                }
11515            },
11516            {
11517                "type": "SignalApplied",
11518                "payload": {
11519                    "sequence": 3,
11520                    "signal_id": "signal-3",
11521                    "signal_name": "increment",
11522                    "payload_codec": "json",
11523                    "value": {"codec": "json", "blob": "[3]"}
11524                }
11525            }
11526        ]);
11527
11528        let running = worker
11529            .execute_query_task(replay_counter_query(
11530                "current",
11531                running_history.clone(),
11532                "running",
11533            ))
11534            .await
11535            .expect("running replay query");
11536        assert_eq!(
11537            running.clone().into_json().expect("query projection"),
11538            json!({"loaded": "loaded", "count": 3, "finished": false})
11539        );
11540
11541        let detached = worker
11542            .execute_query_task(replay_counter_query(
11543                "detached-mutation",
11544                running_history.clone(),
11545                "running",
11546            ))
11547            .await
11548            .expect("query mutates only its detached state clone");
11549        assert_eq!(detached.into_json().expect("query projection"), json!(999));
11550        let failed = worker
11551            .execute_query_task(replay_counter_query(
11552                "failed-mutation",
11553                running_history.clone(),
11554                "running",
11555            ))
11556            .await
11557            .expect_err("failed query");
11558        assert_eq!(failed.reason, "query_rejected");
11559        let unchanged = worker
11560            .execute_query_task(replay_counter_query("current", running_history, "running"))
11561            .await
11562            .expect("later query reconstructs unchanged state");
11563        assert_eq!(unchanged, running);
11564
11565        let restarted_worker = replay_counter_worker();
11566        let restarted_task: QueryTask = serde_json::from_value(json!({
11567            "query_task_id": "query-after-restart",
11568            "workflow_id": "counter-1",
11569            "run_id": "run-counter-1",
11570            "workflow_type": "replay-counter",
11571            "query_name": "current",
11572            "payload_codec": "json",
11573            "workflow_arguments": {"codec": "json", "blob": "[]"},
11574            "query_arguments": {"codec": "json", "blob": "[]"},
11575            "history_events": [],
11576            "history_export": {
11577                "payloads": {"codec": "json"},
11578                "history_events": [
11579                    {
11580                        "type": "ActivityCompleted",
11581                        "payload": {
11582                            "sequence": 1,
11583                            "activity_type": "load-counter",
11584                            "payload_codec": "json",
11585                            "result": null
11586                        }
11587                    },
11588                    {
11589                        "type": "SignalWaitOpened",
11590                        "payload": {
11591                            "sequence": 3,
11592                            "signal_name": "increment"
11593                        }
11594                    },
11595                    {
11596                        "type": "SignalReceived",
11597                        "payload": {
11598                            "signal_id": "signal-3",
11599                            "signal_name": "increment",
11600                            "workflow_sequence": 2
11601                        }
11602                    },
11603                    {
11604                        "type": "SignalApplied",
11605                        "payload": {
11606                            "sequence": 3,
11607                            "signal_id": "signal-3",
11608                            "signal_name": "increment"
11609                        }
11610                    },
11611                    {
11612                        "type": "SignalWaitOpened",
11613                        "payload": {
11614                            "sequence": 5,
11615                            "signal_name": "increment"
11616                        }
11617                    },
11618                    {
11619                        "type": "SignalReceived",
11620                        "payload": {
11621                            "signal_id": "signal-5",
11622                            "signal_name": "increment",
11623                            "workflow_sequence": 4
11624                        }
11625                    },
11626                    {
11627                        "type": "SignalApplied",
11628                        "payload": {
11629                            "sequence": 5,
11630                            "signal_id": "signal-5",
11631                            "signal_name": "increment"
11632                        }
11633                    }
11634                ],
11635                "activities": [{
11636                    "sequence": 1,
11637                    "activity_type": "load-counter",
11638                    "payload_codec": "json",
11639                    "result": {"codec": "json", "blob": "\"loaded\""}
11640                }],
11641                "signals": [
11642                    {
11643                        "id": "signal-3",
11644                        "name": "increment",
11645                        "workflow_sequence": 2,
11646                        "payload_codec": "json",
11647                        "arguments": "[3]"
11648                    },
11649                    {
11650                        "id": "signal-5",
11651                        "name": "increment",
11652                        "workflow_sequence": 4,
11653                        "payload_codec": "json",
11654                        "arguments": "[5]"
11655                    }
11656                ]
11657            },
11658            "run_status": "completed"
11659        }))
11660        .expect("cold replay query task");
11661        let completed = restarted_worker
11662            .execute_query_task(restarted_task)
11663            .await
11664            .expect("completed cold replay query");
11665        assert_eq!(
11666            completed.into_json().expect("query projection"),
11667            json!({"loaded": "loaded", "count": 8, "finished": true})
11668        );
11669    }
11670
11671    #[tokio::test]
11672    async fn replayed_query_replay_failures_are_machine_readable() {
11673        let worker = replay_counter_worker();
11674        let task = replay_counter_query(
11675            "current",
11676            json!([{
11677                "type": "ActivityCompleted",
11678                "payload": {
11679                    "sequence": 1,
11680                    "payload_codec": "json",
11681                    "result": {"codec": "json", "blob": "{"}
11682                }
11683            }]),
11684            "running",
11685        );
11686        let failure = worker
11687            .execute_query_task(task)
11688            .await
11689            .expect_err("invalid replay history payload");
11690        assert_eq!(failure.reason, "query_workflow_state_unavailable");
11691        assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11692    }
11693
11694    #[tokio::test]
11695    async fn query_task_restores_compact_history_from_export() {
11696        let client = Client::new("http://127.0.0.1:8080").expect("client");
11697        let mut worker = Worker::new(client, "rust-workers");
11698        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11699        worker.register_query("counter", "current", |ctx, _args| async move {
11700            Ok(json!(ctx.signals("increment")[0][0]))
11701        });
11702        let task: QueryTask = serde_json::from_value(json!({
11703            "query_task_id": "query-export",
11704            "workflow_type": "counter",
11705            "query_name": "current",
11706            "payload_codec": "json",
11707            "workflow_arguments": {"codec": "json", "blob": "[]"},
11708            "query_arguments": {"codec": "json", "blob": "[]"},
11709            "history_events": [],
11710            "history_export": {
11711                "payloads": {"codec": "json"},
11712                "history_events": [{
11713                    "type": "SignalReceived",
11714                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11715                }],
11716                "signals": [{
11717                    "id": "signal-export",
11718                    "name": "increment",
11719                    "status": "applied",
11720                    "workflow_sequence": 1,
11721                    "payload_codec": "json",
11722                    "arguments": "[9]"
11723                }]
11724            }
11725        }))
11726        .expect("query task");
11727
11728        let result = worker.execute_query_task(task).await.expect("query result");
11729        assert_eq!(result.into_json().expect("query projection"), json!(9));
11730    }
11731
11732    #[tokio::test]
11733    async fn query_task_failures_have_stable_reasons() {
11734        let client = Client::new("http://127.0.0.1:8080").expect("client");
11735        let mut worker = Worker::new(client, "rust-workers");
11736        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11737        worker.register_query(
11738            "counter",
11739            "current",
11740            |_ctx, _args| async move { Ok(json!(0)) },
11741        );
11742
11743        let base_task = QueryTask {
11744            query_task_id: "query-errors".to_string(),
11745            query_task_attempt: 1,
11746            lease_owner: None,
11747            workflow_id: Some("counter-errors".to_string()),
11748            run_id: Some("run-errors".to_string()),
11749            workflow_type: "counter".to_string(),
11750            query_name: "missing".to_string(),
11751            payload_codec: JSON_CODEC.to_string(),
11752            workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11753            query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11754            history_events: Vec::new(),
11755            history_export: None,
11756            run_status: Some("running".to_string()),
11757        };
11758
11759        let unknown = worker
11760            .execute_query_task(base_task.clone())
11761            .await
11762            .expect_err("unknown query");
11763        assert_eq!(unknown.reason, "rejected_unknown_query");
11764
11765        let mut malformed = base_task;
11766        malformed.query_name = "current".to_string();
11767        malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11768        let malformed = worker
11769            .execute_query_task(malformed)
11770            .await
11771            .expect_err("malformed payload");
11772        assert_eq!(malformed.reason, "query_payload_decode_failed");
11773
11774        let client = Client::new("http://127.0.0.1:8080").expect("client");
11775        let mut unavailable_worker = Worker::new(client, "rust-workers");
11776        unavailable_worker
11777            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11778        let unavailable_task: QueryTask = serde_json::from_value(json!({
11779            "query_task_id": "query-unavailable",
11780            "workflow_type": "counter",
11781            "query_name": "current",
11782            "payload_codec": "json",
11783            "workflow_arguments": {"codec": "json", "blob": "[]"},
11784            "query_arguments": {"codec": "json", "blob": "[]"}
11785        }))
11786        .expect("query task");
11787        let unavailable = unavailable_worker
11788            .execute_query_task(unavailable_task)
11789            .await
11790            .expect_err("query handler unavailable");
11791        assert_eq!(unavailable.reason, "query_handler_unavailable");
11792    }
11793
11794    #[tokio::test]
11795    async fn client_query_decodes_result_and_typed_failure() {
11796        let server = MockWorkerServer::start();
11797        let client = Client::builder(server.base_url())
11798            .timeout(Duration::from_secs(2))
11799            .build()
11800            .expect("client");
11801
11802        let result = client
11803            .query_workflow("counter-1", "current", json!([]))
11804            .await
11805            .expect("query result");
11806        assert_eq!(result, json!({"count": 8}));
11807
11808        let error = client
11809            .query_workflow("counter-1", "missing", json!([]))
11810            .await
11811            .expect_err("unknown query");
11812        let Error::QueryFailed(failure) = error else {
11813            panic!("expected typed query failure");
11814        };
11815        assert_eq!(failure.status, 404);
11816        assert_eq!(failure.reason, "rejected_unknown_query");
11817    }
11818
11819    #[tokio::test]
11820    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11821        let server = MockWorkerServer::start();
11822        let client = Client::builder(server.base_url())
11823            .timeout(Duration::from_secs(2))
11824            .build()
11825            .expect("client");
11826        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11827
11828        client
11829            .start_workflow(
11830                "typed.echo",
11831                "rust-workers",
11832                "typed-start",
11833                arguments.clone(),
11834            )
11835            .await
11836            .expect("typed workflow start");
11837        assert_eq!(
11838            decode_wire_avro_value(
11839                &server.request_body("/api/workflows")["input"],
11840                DEFAULT_CODEC,
11841            )
11842            .expect("typed start input"),
11843            arguments
11844        );
11845
11846        client
11847            .signal_workflow("typed-1", "changed", arguments.clone())
11848            .await
11849            .expect("typed signal");
11850        assert_eq!(
11851            decode_wire_avro_value(
11852                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11853                DEFAULT_CODEC,
11854            )
11855            .expect("typed signal input"),
11856            arguments
11857        );
11858
11859        assert_eq!(
11860            client
11861                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11862                .await
11863                .expect("typed query"),
11864            typed_fidelity_probe()
11865        );
11866        assert_eq!(
11867            decode_wire_avro_value(
11868                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11869                DEFAULT_CODEC,
11870            )
11871            .expect("typed query input"),
11872            arguments
11873        );
11874
11875        assert_eq!(
11876            client
11877                .update_workflow_avro_value(
11878                    "typed-1",
11879                    "replace",
11880                    arguments.clone(),
11881                    Some("typed-request"),
11882                )
11883                .await
11884                .expect("typed update"),
11885            typed_fidelity_probe()
11886        );
11887        let update = server.request_body("/api/workflows/typed-1/update/replace");
11888        assert_eq!(update["request_id"], "typed-request");
11889        assert_eq!(
11890            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11891            arguments
11892        );
11893
11894        let handle = WorkflowHandle {
11895            client: client.clone(),
11896            workflow_id: "typed-1".to_string(),
11897            run_id: Some("run-typed-1".to_string()),
11898            workflow_type: "typed.echo".to_string(),
11899        };
11900        assert_eq!(
11901            handle
11902                .result_avro_value(WorkflowResultOptions::default())
11903                .await
11904                .expect("typed workflow result"),
11905            typed_fidelity_probe()
11906        );
11907
11908        client
11909            .complete_activity_task(
11910                "activity-typed",
11911                "attempt-typed",
11912                "rust-worker",
11913                typed_fidelity_probe(),
11914                DEFAULT_CODEC,
11915            )
11916            .await
11917            .expect("typed activity completion");
11918        assert_eq!(
11919            decode_wire_avro_value(
11920                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11921                    ["result"],
11922                DEFAULT_CODEC,
11923            )
11924            .expect("typed activity result"),
11925            typed_fidelity_probe()
11926        );
11927        client
11928            .fail_activity_task(
11929                "activity-typed",
11930                "attempt-typed",
11931                "rust-worker",
11932                "typed failure",
11933                true,
11934            )
11935            .await
11936            .expect("activity failure");
11937    }
11938
11939    #[tokio::test]
11940    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11941        let server = MockWorkerServer::start();
11942        let client = Client::builder(server.base_url())
11943            .timeout(Duration::from_secs(2))
11944            .build()
11945            .expect("client");
11946
11947        let options = WorkflowCommandOptions::new()
11948            .reason("cleanup requested")
11949            .request_id("cancel-17");
11950        let cancelled = client
11951            .cancel_workflow("wf-lifecycle", options)
11952            .await
11953            .expect("instance cancellation");
11954        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11955        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11956        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11957        assert_eq!(
11958            server.request_body("/api/workflows/wf-lifecycle/cancel"),
11959            json!({"reason":"cleanup requested","request_id":"cancel-17"})
11960        );
11961
11962        let terminated = client
11963            .terminate_workflow(
11964                "wf-lifecycle",
11965                WorkflowCommandOptions::new().reason("forced stop"),
11966            )
11967            .await
11968            .expect("instance termination");
11969        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11970        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11971
11972        client
11973            .cancel_workflow_run(
11974                "wf-lifecycle",
11975                "run-current",
11976                WorkflowCommandOptions::default(),
11977            )
11978            .await
11979            .expect("selected run cancellation");
11980        client
11981            .terminate_workflow_run(
11982                "wf-lifecycle",
11983                "run-current",
11984                WorkflowCommandOptions::default(),
11985            )
11986            .await
11987            .expect("selected run termination");
11988
11989        for (command, error) in [
11990            (
11991                WorkflowCommandKind::Cancel,
11992                client
11993                    .cancel_workflow_run(
11994                        "wf-lifecycle",
11995                        "run-stale",
11996                        WorkflowCommandOptions::default(),
11997                    )
11998                    .await
11999                    .expect_err("stale cancellation must be rejected"),
12000            ),
12001            (
12002                WorkflowCommandKind::Terminate,
12003                client
12004                    .terminate_workflow_run(
12005                        "wf-lifecycle",
12006                        "run-stale",
12007                        WorkflowCommandOptions::default(),
12008                    )
12009                    .await
12010                    .expect_err("stale termination must be rejected"),
12011            ),
12012        ] {
12013            let Error::WorkflowCommandRejected(rejection) = error else {
12014                panic!("expected typed command rejection");
12015            };
12016            assert_eq!(rejection.command, command);
12017            assert_eq!(rejection.status, 409);
12018            assert_eq!(rejection.reason, "historical_run_command_rejected");
12019            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12020            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12021        }
12022    }
12023
12024    #[tokio::test]
12025    async fn workflow_start_options_send_server_enforced_deadlines() {
12026        let server = MockWorkerServer::start();
12027        let client = Client::builder(server.base_url())
12028            .timeout(Duration::from_secs(2))
12029            .build()
12030            .expect("client");
12031
12032        let handle = client
12033            .start_workflow_with_options(
12034                "rust.timeout",
12035                "rust-timeouts",
12036                "wf-start-options",
12037                WorkflowStartOptions::new()
12038                    .execution_timeout_seconds(30)
12039                    .run_timeout_seconds(1),
12040                json!([]),
12041            )
12042            .await
12043            .expect("workflow start");
12044
12045        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12046        let body = server.request_body("/api/workflows");
12047        assert_eq!(body["execution_timeout_seconds"], 30);
12048        assert_eq!(body["run_timeout_seconds"], 1);
12049
12050        let invalid = client
12051            .start_workflow_with_options(
12052                "rust.timeout",
12053                "rust-timeouts",
12054                "wf-invalid-options",
12055                WorkflowStartOptions::new()
12056                    .execution_timeout_seconds(1)
12057                    .run_timeout_seconds(2),
12058                json!([]),
12059            )
12060            .await
12061            .expect_err("invalid deadline ordering");
12062        assert!(invalid
12063            .to_string()
12064            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12065    }
12066
12067    #[tokio::test]
12068    async fn workflow_result_returns_each_typed_terminal_outcome() {
12069        let server = MockWorkerServer::start();
12070        let client = Client::builder(server.base_url())
12071            .timeout(Duration::from_secs(2))
12072            .build()
12073            .expect("client");
12074        let options = WorkflowResultOptions {
12075            poll_interval: Duration::ZERO,
12076            timeout: Duration::from_secs(1),
12077        };
12078
12079        let failed = WorkflowHandle {
12080            client: client.clone(),
12081            workflow_id: "wf-failed".to_string(),
12082            run_id: Some("run-failed".to_string()),
12083            workflow_type: "failure".to_string(),
12084        }
12085        .result(options)
12086        .await
12087        .expect_err("failed outcome");
12088        let Error::WorkflowFailed(failure) = failed else {
12089            panic!("expected WorkflowFailed");
12090        };
12091        assert_eq!(failure.workflow_id, "wf-failed");
12092        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12093        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12094        assert_eq!(failure.failure_category.as_deref(), Some("application"));
12095        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12096        assert_eq!(
12097            failure.exception_class.as_deref(),
12098            Some("billing::PaymentError")
12099        );
12100        assert_eq!(failure.non_retryable, Some(true));
12101
12102        for (workflow_id, expected_kind, expected_reason) in [
12103            (
12104                "wf-cancelled",
12105                WorkflowTerminalKind::Cancelled,
12106                "cleanup requested",
12107            ),
12108            (
12109                "wf-terminated",
12110                WorkflowTerminalKind::Terminated,
12111                "forced stop",
12112            ),
12113            (
12114                "wf-timed-out",
12115                WorkflowTerminalKind::TimedOut,
12116                "run_timeout",
12117            ),
12118        ] {
12119            let error = WorkflowHandle {
12120                client: client.clone(),
12121                workflow_id: workflow_id.to_string(),
12122                run_id: None,
12123                workflow_type: "terminal".to_string(),
12124            }
12125            .result(options)
12126            .await
12127            .expect_err("typed terminal outcome");
12128            let outcome = match error {
12129                Error::WorkflowCancelled(outcome) => outcome,
12130                Error::WorkflowTerminated(outcome) => outcome,
12131                Error::WorkflowTimedOut(outcome) => outcome,
12132                other => panic!("unexpected terminal error: {other}"),
12133            };
12134            assert_eq!(outcome.kind, expected_kind);
12135            assert_eq!(outcome.workflow_id, workflow_id);
12136            assert_eq!(outcome.reason, expected_reason);
12137        }
12138
12139        let wait_timeout = WorkflowHandle {
12140            client,
12141            workflow_id: "wf-waiting".to_string(),
12142            run_id: Some("run-waiting".to_string()),
12143            workflow_type: "waiting".to_string(),
12144        }
12145        .result(WorkflowResultOptions {
12146            poll_interval: Duration::ZERO,
12147            timeout: Duration::ZERO,
12148        })
12149        .await
12150        .expect_err("client wait timeout");
12151        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12152            panic!("expected typed client timeout");
12153        };
12154        assert_eq!(timeout.reason, "result_wait_timeout");
12155        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12156        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12157    }
12158
12159    #[tokio::test]
12160    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12161        let server = MockWorkerServer::start();
12162        let client = Client::builder(server.base_url())
12163            .timeout(Duration::from_secs(2))
12164            .build()
12165            .expect("client");
12166
12167        let handle = WorkflowHandle {
12168            client,
12169            workflow_id: "wf-selected".to_string(),
12170            run_id: Some("run-selected".to_string()),
12171            workflow_type: "selected".to_string(),
12172        };
12173        let options = WorkflowResultOptions {
12174            poll_interval: Duration::ZERO,
12175            timeout: Duration::from_secs(1),
12176        };
12177
12178        let current = handle
12179            .result(options)
12180            .await
12181            .expect("instance result follows the current run");
12182        assert_eq!(current, json!("current run output"));
12183
12184        let error = handle
12185            .result_selected_run(options)
12186            .await
12187            .expect_err("the selected run is cancelled even though the current run completed");
12188
12189        let Error::WorkflowCancelled(outcome) = error else {
12190            panic!("expected selected run cancellation");
12191        };
12192        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12193        assert_eq!(outcome.reason, "selected run cancelled");
12194        assert_eq!(
12195            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12196            1
12197        );
12198        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12199    }
12200
12201    #[tokio::test]
12202    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12203        let server = MockWorkerServer::draining_polls();
12204        let client = Client::builder(server.base_url())
12205            .timeout(Duration::from_secs(2))
12206            .build()
12207            .expect("client");
12208
12209        let workflow = client
12210            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12211            .await
12212            .expect("workflow drain response");
12213        let activity = client
12214            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12215            .await
12216            .expect("activity drain response");
12217        let query = client
12218            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12219            .await
12220            .expect("query drain response");
12221
12222        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12223            assert_eq!(
12224                outcome,
12225                WorkerPollOutcome::Stop {
12226                    poll_status: Some("draining".to_string()),
12227                    reason: Some("worker_draining".to_string()),
12228                }
12229            );
12230        }
12231
12232        assert!(client
12233            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12234            .await
12235            .expect("compatibility poll")
12236            .is_none());
12237    }
12238
12239    #[tokio::test]
12240    async fn managed_worker_honors_drain_stop_for_every_task_family() {
12241        let server = MockWorkerServer::draining_polls();
12242        let client = Client::builder(server.base_url())
12243            .timeout(Duration::from_secs(2))
12244            .build()
12245            .expect("client");
12246
12247        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12248            .worker_id("draining-workflow-worker")
12249            .poll_timeout(Duration::ZERO);
12250        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12251        workflow_worker
12252            .run()
12253            .await
12254            .expect("workflow drain is a clean stop");
12255
12256        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12257            .worker_id("draining-activity-worker")
12258            .poll_timeout(Duration::ZERO);
12259        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12260        activity_worker
12261            .run()
12262            .await
12263            .expect("activity drain is a clean stop");
12264
12265        let mut query_worker = Worker::new(client, "rust-workers")
12266            .worker_id("draining-query-worker")
12267            .poll_timeout(Duration::ZERO);
12268        query_worker.register_query("counter", "current", |_ctx, _args| async {
12269            Ok(Value::Null)
12270        });
12271        query_worker
12272            .run()
12273            .await
12274            .expect("query drain is a clean stop");
12275    }
12276
12277    #[tokio::test]
12278    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12279        let server = MockWorkerServer::start();
12280        let client = Client::builder(server.base_url())
12281            .timeout(Duration::from_secs(2))
12282            .build()
12283            .expect("client");
12284
12285        let heartbeat = client
12286            .heartbeat_activity_task(
12287                "activity-cancel",
12288                "attempt-cancel",
12289                "rust-worker",
12290                typed_fidelity_probe(),
12291            )
12292            .await
12293            .expect("cancellation heartbeat");
12294        assert!(heartbeat.cancel_requested);
12295        assert!(heartbeat.should_stop());
12296        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12297        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12298        let heartbeat_body =
12299            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12300        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12301        assert_eq!(
12302            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12303                .expect("typed heartbeat details"),
12304            typed_fidelity_probe()
12305        );
12306
12307        let error = client
12308            .complete_activity_task(
12309                "activity-cancel",
12310                "attempt-cancel",
12311                "rust-worker",
12312                json!({"late":true}),
12313                JSON_CODEC,
12314            )
12315            .await
12316            .expect_err("late completion must be refused");
12317        assert!(activity_task_rejection_is_final(&error));
12318        let Error::ActivityTaskRejected(rejection) = error else {
12319            panic!("expected typed activity rejection");
12320        };
12321        assert_eq!(rejection.status, 409);
12322        assert_eq!(rejection.reason, "run_cancelled");
12323        assert!(rejection.cancel_requested);
12324        assert_eq!(rejection.can_continue, Some(false));
12325    }
12326
12327    #[tokio::test]
12328    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12329        let server = MockWorkerServer::cancelled_activity();
12330        let client = Client::builder(server.base_url())
12331            .timeout(Duration::from_secs(2))
12332            .build()
12333            .expect("client");
12334        let cancellation_observed = Arc::new(AtomicBool::new(false));
12335        let observed = Arc::clone(&cancellation_observed);
12336        let mut worker = Worker::new(client.clone(), "rust-workers")
12337            .worker_id("rust-cancel-worker")
12338            .poll_timeout(Duration::from_millis(10));
12339        worker.register_activity("cancel-aware", move |ctx, _args| {
12340            let observed = Arc::clone(&observed);
12341            async move {
12342                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12343                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12344                Ok(json!({"late":"completion"}))
12345            }
12346        });
12347
12348        assert_eq!(
12349            worker.run_once().await.expect("cancelled attempt handled"),
12350            1
12351        );
12352        assert!(cancellation_observed.load(Ordering::SeqCst));
12353        assert_eq!(
12354            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12355            1
12356        );
12357
12358        let mut restarted = Worker::new(client, "rust-workers")
12359            .worker_id("rust-cancel-worker-restarted")
12360            .poll_timeout(Duration::from_millis(10));
12361        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12362        assert_eq!(
12363            restarted
12364                .run_once()
12365                .await
12366                .expect("replacement worker continues polling"),
12367            0
12368        );
12369    }
12370
12371    #[tokio::test]
12372    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12373        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12374        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12375        let client = Client::builder(server.base_url())
12376            .timeout(Duration::from_secs(2))
12377            .build()
12378            .expect("client");
12379
12380        let direct_error = client
12381            .complete_workflow_task(
12382                "workflow-timeout-task",
12383                "timeout-worker",
12384                3,
12385                vec![json!({"type": "complete_workflow", "result": null})],
12386            )
12387            .await
12388            .expect_err("the low-level client preserves the completion rejection");
12389        let Error::Http { status, body } = direct_error else {
12390            panic!("expected the original HTTP completion rejection");
12391        };
12392        assert_eq!(status, reqwest::StatusCode::CONFLICT);
12393        assert_eq!(
12394            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12395            "run_timed_out"
12396        );
12397
12398        let mut worker = Worker::new(client, "rust-workers")
12399            .worker_id("timeout-worker")
12400            .poll_timeout(Duration::from_millis(10));
12401        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12402            Ok(json!({"late": "result"}))
12403        });
12404
12405        assert_eq!(
12406            worker
12407                .run_once()
12408                .await
12409                .expect("authoritative selected-run timeout settles the tick"),
12410            1
12411        );
12412        assert_eq!(
12413            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12414            2,
12415            "both the direct client proof and managed worker must see the rejection"
12416        );
12417    }
12418
12419    #[tokio::test]
12420    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12421        for (name, status, response) in [
12422            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12423            (
12424                "command was recorded",
12425                "409 Conflict",
12426                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12427            ),
12428            (
12429                "lease conflict",
12430                "409 Conflict",
12431                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12432            ),
12433            (
12434                "nonterminal run",
12435                "409 Conflict",
12436                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12437            ),
12438            (
12439                "different selected run",
12440                "409 Conflict",
12441                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12442            ),
12443            (
12444                "different task attempt",
12445                "409 Conflict",
12446                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12447            ),
12448            (
12449                "authentication failure",
12450                "401 Unauthorized",
12451                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12452            ),
12453            (
12454                "authorization failure",
12455                "403 Forbidden",
12456                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12457            ),
12458            (
12459                "protocol failure",
12460                "400 Bad Request",
12461                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12462            ),
12463            (
12464                "malformed command",
12465                "422 Unprocessable Entity",
12466                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12467            ),
12468            (
12469                "transient server failure",
12470                "503 Service Unavailable",
12471                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12472            ),
12473        ] {
12474            let server = MockWorkerServer::workflow_completion(status, response);
12475            let client = Client::builder(server.base_url())
12476                .timeout(Duration::from_secs(2))
12477                .build()
12478                .expect("client");
12479            let mut worker = Worker::new(client, "rust-workers")
12480                .worker_id("timeout-worker")
12481                .poll_timeout(Duration::from_millis(10));
12482            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12483                Ok(json!({"late": "result"}))
12484            });
12485
12486            let error = worker
12487                .run_once()
12488                .await
12489                .expect_err(&format!("{name} must remain an error"));
12490            assert!(
12491                matches!(error, Error::Http { .. } | Error::Protocol(_)),
12492                "{name} returned an unexpected error variant: {error}"
12493            );
12494        }
12495    }
12496
12497    #[tokio::test]
12498    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
12499        let server = MockWorkerServer::start();
12500        let client = Client::builder(server.base_url())
12501            .worker_token(Some("worker-secret".to_string()))
12502            .namespace("orders")
12503            .timeout(Duration::from_secs(2))
12504            .build()
12505            .expect("client");
12506        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
12507
12508        let result = client
12509            .deregister_worker_registration("worker/α space")
12510            .await
12511            .expect("deregister worker registration");
12512
12513        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
12514        assert_eq!(
12515            server.worker_protocol_for(path).as_deref(),
12516            Some(WORKER_PROTOCOL_VERSION)
12517        );
12518        assert_eq!(server.control_protocol_for(path), None);
12519        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
12520        assert_eq!(
12521            server.authorization_for(path).as_deref(),
12522            Some("Bearer worker-secret")
12523        );
12524        assert_eq!(
12525            result,
12526            WorkerDeregistrationEnvelope {
12527                worker_id: "deregistered-worker".to_string(),
12528                outcome: "deregistered".to_string(),
12529                recovered_workflow_task_count: 2,
12530            }
12531        );
12532    }
12533
12534    #[tokio::test]
12535    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
12536        let server = MockWorkerServer::start();
12537        let control_only = Client::builder(server.base_url())
12538            .control_token(Some("control-secret".to_string()))
12539            .build()
12540            .expect("control client");
12541
12542        let error = control_only
12543            .register_worker("worker", "queue", vec![], vec![], 1, 1)
12544            .await
12545            .expect_err("control token must not authorize a worker request");
12546        assert!(matches!(
12547            error,
12548            Error::MissingRoleCredentials { role: "worker", .. }
12549        ));
12550        assert_eq!(server.request_count("/api/worker/register"), 0);
12551
12552        let worker_only = Client::builder(server.base_url())
12553            .worker_token(Some("worker-secret".to_string()))
12554            .build()
12555            .expect("worker client");
12556        let error = worker_only
12557            .health()
12558            .await
12559            .expect_err("worker token must not authorize a control request");
12560        assert!(matches!(
12561            error,
12562            Error::MissingRoleCredentials {
12563                role: "control",
12564                ..
12565            }
12566        ));
12567        assert_eq!(server.request_count("/api/health"), 0);
12568    }
12569
12570    #[tokio::test]
12571    async fn shared_token_supports_worker_and_control_planes() {
12572        let server = MockWorkerServer::start();
12573        let client = Client::builder(server.base_url())
12574            .token(Some("shared-secret".to_string()))
12575            .build()
12576            .expect("client");
12577
12578        client.health().await.expect("control request");
12579        client
12580            .register_worker("worker", "queue", vec![], vec![], 1, 1)
12581            .await
12582            .expect("worker request");
12583
12584        assert_eq!(
12585            server.authorization_for("/api/health").as_deref(),
12586            Some("Bearer shared-secret")
12587        );
12588        assert_eq!(
12589            server.control_protocol_for("/api/health").as_deref(),
12590            Some(CONTROL_PLANE_VERSION)
12591        );
12592        assert_eq!(
12593            server.authorization_for("/api/worker/register").as_deref(),
12594            Some("Bearer shared-secret")
12595        );
12596        assert_eq!(
12597            server
12598                .worker_protocol_for("/api/worker/register")
12599                .as_deref(),
12600            Some(WORKER_PROTOCOL_VERSION)
12601        );
12602    }
12603
12604    #[tokio::test]
12605    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12606        let server = MockWorkerServer::start();
12607        let client = Client::builder(server.base_url())
12608            .timeout(Duration::from_secs(2))
12609            .build()
12610            .expect("client");
12611
12612        client
12613            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12614            .await
12615            .expect("register");
12616        client
12617            .heartbeat_worker("capture-worker", 1, 1)
12618            .await
12619            .expect("heartbeat");
12620        client
12621            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12622            .await
12623            .expect("workflow poll");
12624        client
12625            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12626            .await
12627            .expect("activity poll");
12628
12629        for path in [
12630            "/api/worker/register",
12631            "/api/worker/heartbeat",
12632            "/api/worker/workflow-tasks/poll",
12633            "/api/worker/activity-tasks/poll",
12634        ] {
12635            assert_eq!(
12636                server.worker_protocol_for(path).as_deref(),
12637                Some(WORKER_PROTOCOL_VERSION),
12638                "unexpected protocol for {path}"
12639            );
12640        }
12641
12642        assert_eq!(
12643            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12644            1
12645        );
12646        assert_eq!(
12647            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12648            1
12649        );
12650        assert!(
12651            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12652                .as_str()
12653                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12654        );
12655        assert!(
12656            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12657                .as_str()
12658                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12659        );
12660    }
12661
12662    #[tokio::test]
12663    async fn query_task_endpoints_send_the_query_feature_protocol() {
12664        let server = MockWorkerServer::start();
12665        let client = Client::builder(server.base_url())
12666            .timeout(Duration::from_secs(2))
12667            .build()
12668            .expect("client");
12669
12670        client
12671            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12672            .await
12673            .expect("query poll");
12674        client
12675            .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12676            .await
12677            .expect("query complete");
12678        client
12679            .fail_query_task(
12680                "query-capture",
12681                "capture-worker",
12682                1,
12683                "failed",
12684                "query_rejected",
12685                "QueryFailed",
12686            )
12687            .await
12688            .expect("query fail");
12689
12690        for path in [
12691            "/api/worker/query-tasks/poll",
12692            "/api/worker/query-tasks/query-capture/complete",
12693            "/api/worker/query-tasks/query-capture/fail",
12694        ] {
12695            assert_eq!(
12696                server.worker_protocol_for(path).as_deref(),
12697                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12698                "unexpected protocol for {path}"
12699            );
12700        }
12701
12702        assert_eq!(
12703            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12704            1
12705        );
12706        assert!(
12707            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12708                .as_str()
12709                .is_some_and(|id| id.starts_with("rust-query-poll-"))
12710        );
12711    }
12712
12713    #[tokio::test]
12714    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12715        let server = MockWorkerServer::transient_worker_failures();
12716        let client = Client::builder(server.base_url())
12717            .timeout(Duration::from_secs(2))
12718            .build()
12719            .expect("client");
12720
12721        client
12722            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12723            .await
12724            .expect("workflow poll retry");
12725        client
12726            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12727            .await
12728            .expect("activity poll retry");
12729        client
12730            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12731            .await
12732            .expect("query poll retry");
12733
12734        for path in [
12735            "/api/worker/workflow-tasks/poll",
12736            "/api/worker/activity-tasks/poll",
12737            "/api/worker/query-tasks/poll",
12738        ] {
12739            let bodies = server.request_bodies(path);
12740            assert_eq!(bodies.len(), 2, "{path} must be retried once");
12741            assert_eq!(
12742                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12743                "{path} must preserve the request binding across retry"
12744            );
12745        }
12746    }
12747
12748    #[tokio::test]
12749    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12750        let server = MockWorkerServer::consecutive_poll_failures(2);
12751        let client = Client::builder(server.base_url())
12752            .timeout(Duration::from_secs(2))
12753            .build()
12754            .expect("client");
12755        let mut worker = Worker::new(client, "capture")
12756            .worker_id("capture-worker")
12757            .poll_timeout(Duration::from_millis(10))
12758            .retry_policy(WorkerRetryPolicy {
12759                max_retries: 2,
12760                initial_backoff: Duration::from_millis(1),
12761                max_backoff: Duration::from_millis(1),
12762            });
12763        worker.register_workflow(
12764            "capture.workflow",
12765            |_ctx, _input| async move { Ok(Value::Null) },
12766        );
12767        worker.register_activity(
12768            "capture.activity",
12769            |_ctx, _input| async move { Ok(Value::Null) },
12770        );
12771        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12772            Ok(Value::Null)
12773        });
12774
12775        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12776
12777        for path in [
12778            "/api/worker/workflow-tasks/poll",
12779            "/api/worker/activity-tasks/poll",
12780            "/api/worker/query-tasks/poll",
12781        ] {
12782            let bodies = server.request_bodies(path);
12783            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12784            assert!(
12785                bodies
12786                    .iter()
12787                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12788                "{path} must preserve one request binding across every retry"
12789            );
12790        }
12791    }
12792
12793    #[tokio::test]
12794    async fn query_protocol_rejection_from_older_server_is_typed() {
12795        let server = MockWorkerServer::reject_query_protocol();
12796        let client = Client::builder(server.base_url())
12797            .timeout(Duration::from_secs(2))
12798            .build()
12799            .expect("client");
12800
12801        let error = client
12802            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12803            .await
12804            .expect_err("server below query protocol floor must reject");
12805        let Error::Protocol(failure) = error else {
12806            panic!("expected typed protocol failure");
12807        };
12808
12809        assert_eq!(failure.status, 400);
12810        assert_eq!(failure.reason, "unsupported_protocol_version");
12811        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12812        assert_eq!(
12813            failure.requested_version.as_deref(),
12814            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12815        );
12816        assert_eq!(
12817            server
12818                .worker_protocol_for("/api/worker/query-tasks/poll")
12819                .as_deref(),
12820            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12821        );
12822    }
12823
12824    #[tokio::test]
12825    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12826        let server = MockWorkerServer::reject_query_protocol();
12827        let client = Client::builder(server.base_url())
12828            .timeout(Duration::from_secs(2))
12829            .build()
12830            .expect("client");
12831        let mut worker = Worker::new(client, "rust-workers")
12832            .worker_id("baseline-worker")
12833            .poll_timeout(Duration::from_millis(10));
12834
12835        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12836            Ok(Value::Null)
12837        });
12838
12839        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12840        assert_eq!(
12841            server
12842                .worker_protocol_for("/api/worker/workflow-tasks/poll")
12843                .as_deref(),
12844            Some(WORKER_PROTOCOL_VERSION)
12845        );
12846        assert_eq!(
12847            server.worker_protocol_for("/api/worker/query-tasks/poll"),
12848            None,
12849            "a worker without query handlers must not use the query-task endpoint"
12850        );
12851    }
12852
12853    #[tokio::test]
12854    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12855        let server = MockWorkerServer::reject_query_completion();
12856        let client = Client::builder(server.base_url())
12857            .timeout(Duration::from_secs(2))
12858            .build()
12859            .expect("client");
12860
12861        let error = client
12862            .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12863            .await
12864            .expect_err("expired completion must be rejected");
12865        let Error::QueryFailed(failure) = error else {
12866            panic!("expected typed query failure");
12867        };
12868        assert_eq!(failure.status, 409);
12869        assert_eq!(failure.reason, "query_task_timed_out");
12870
12871        let mut worker = Worker::new(client, "rust-workers")
12872            .worker_id("late-worker")
12873            .poll_timeout(Duration::from_millis(10));
12874        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12875        worker.register_query(
12876            "counter",
12877            "current",
12878            |_ctx, _args| async move { Ok(json!(8)) },
12879        );
12880
12881        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12882        assert_eq!(
12883            worker
12884                .run_once()
12885                .await
12886                .expect("worker continues after late completion"),
12887            0
12888        );
12889        assert_eq!(
12890            server.request_count("/api/worker/query-tasks/query-late/complete"),
12891            2
12892        );
12893        assert_eq!(
12894            server.request_count("/api/worker/query-tasks/query-late/fail"),
12895            0,
12896            "a server completion rejection must not be reported as an encoding failure"
12897        );
12898    }
12899
12900    #[tokio::test]
12901    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
12902        let server = MockWorkerServer::start();
12903        let client = Client::builder(server.base_url())
12904            .timeout(Duration::from_secs(2))
12905            .build()
12906            .expect("client");
12907        let mut worker = Worker::new(client, "rust-workers")
12908            .worker_id("joined-worker")
12909            .poll_timeout(Duration::from_millis(10));
12910        worker.register_workflow(
12911            "joined.workflow",
12912            |_ctx, _input| async move { Ok(Value::Null) },
12913        );
12914        worker.register_activity(
12915            "joined.activity",
12916            |_ctx, _input| async move { Ok(Value::Null) },
12917        );
12918        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
12919            Ok(Value::Null)
12920        });
12921
12922        worker
12923            .run_until(tokio::time::sleep(Duration::from_millis(20)))
12924            .await
12925            .expect("normal shutdown");
12926
12927        let deregistration_path = "/api/worker/registrations/mock-worker";
12928        assert_eq!(server.request_count(deregistration_path), 1);
12929        for poll_path in [
12930            "/api/worker/workflow-tasks/poll",
12931            "/api/worker/activity-tasks/poll",
12932            "/api/worker/query-tasks/poll",
12933        ] {
12934            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
12935        }
12936        assert_eq!(
12937            server.captured_paths().last().map(String::as_str),
12938            Some(deregistration_path),
12939            "deregistration must start only after every poller has joined"
12940        );
12941    }
12942
12943    #[tokio::test]
12944    async fn registration_failure_does_not_deregister() {
12945        let server = MockWorkerServer::rejected_registration();
12946        let client = Client::builder(server.base_url())
12947            .timeout(Duration::from_secs(2))
12948            .build()
12949            .expect("client");
12950        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
12951
12952        let error = worker
12953            .run_until(async {})
12954            .await
12955            .expect_err("registration must fail");
12956        assert!(matches!(
12957            error,
12958            Error::Http {
12959                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
12960                ..
12961            }
12962        ));
12963        assert!(server
12964            .captured_paths()
12965            .iter()
12966            .all(|path| !path.starts_with("/api/worker/registrations/")));
12967    }
12968
12969    #[tokio::test]
12970    async fn declined_registration_does_not_deregister() {
12971        let server = MockWorkerServer::declined_registration();
12972        let client = Client::builder(server.base_url())
12973            .timeout(Duration::from_secs(2))
12974            .build()
12975            .expect("client");
12976        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
12977
12978        let error = worker
12979            .run_until(async {})
12980            .await
12981            .expect_err("declined registration must fail");
12982        assert!(matches!(error, Error::WorkerLoop(_)));
12983        assert!(error.to_string().contains("was not accepted"));
12984        assert!(server
12985            .captured_paths()
12986            .iter()
12987            .all(|path| !path.starts_with("/api/worker/registrations/")));
12988    }
12989
12990    #[tokio::test]
12991    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
12992        let server = MockWorkerServer::rejected_deregistration();
12993        let client = Client::builder(server.base_url())
12994            .timeout(Duration::from_secs(2))
12995            .build()
12996            .expect("client");
12997        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
12998
12999        let error = worker
13000            .run_until(async {})
13001            .await
13002            .expect_err("deregistration must fail");
13003        assert!(matches!(
13004            error,
13005            Error::Http {
13006                status: reqwest::StatusCode::FORBIDDEN,
13007                ..
13008            }
13009        ));
13010        assert_eq!(
13011            server.request_count("/api/worker/registrations/mock-worker"),
13012            1
13013        );
13014    }
13015
13016    #[tokio::test]
13017    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13018        let server = MockWorkerServer::rejected_deregistration_protocol();
13019        let client = Client::builder(server.base_url())
13020            .timeout(Duration::from_secs(2))
13021            .build()
13022            .expect("client");
13023        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13024
13025        let error = worker
13026            .run_until(async {})
13027            .await
13028            .expect_err("protocol rejection must fail shutdown");
13029        let Error::Protocol(failure) = error else {
13030            panic!("expected typed protocol failure");
13031        };
13032        assert_eq!(failure.reason, "unsupported_protocol_version");
13033        assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13034        assert_eq!(
13035            server.request_count("/api/worker/registrations/mock-worker"),
13036            1
13037        );
13038    }
13039
13040    #[tokio::test]
13041    async fn primary_poller_error_retains_deregistration_failure_context() {
13042        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13043        let client = Client::builder(server.base_url())
13044            .timeout(Duration::from_secs(2))
13045            .build()
13046            .expect("client");
13047        let mut worker = Worker::new(client, "rust-workers")
13048            .worker_id("combined-failure")
13049            .poll_timeout(Duration::from_millis(10));
13050        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13051            Ok(Value::Null)
13052        });
13053
13054        let error = worker
13055            .run()
13056            .await
13057            .expect_err("worker and cleanup must fail");
13058        let summary = error.to_string();
13059        assert!(summary.contains("authentication_failed"));
13060        assert!(summary.contains("worker cannot deregister"));
13061        let Error::WorkerShutdown {
13062            primary,
13063            deregistration,
13064        } = error
13065        else {
13066            panic!("expected combined worker shutdown error");
13067        };
13068        assert!(matches!(
13069            *primary,
13070            Error::Http {
13071                status: reqwest::StatusCode::UNAUTHORIZED,
13072                ..
13073            }
13074        ));
13075        assert!(matches!(
13076            *deregistration,
13077            Error::Http {
13078                status: reqwest::StatusCode::FORBIDDEN,
13079                ..
13080            }
13081        ));
13082        assert_eq!(
13083            server.request_count("/api/worker/registrations/mock-worker"),
13084            1
13085        );
13086    }
13087
13088    #[tokio::test]
13089    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
13090        let server = MockWorkerServer::start();
13091        let client = Client::builder(server.base_url())
13092            .timeout(Duration::from_secs(2))
13093            .build()
13094            .expect("client");
13095        let mut worker = Worker::new(client, "rust-workers")
13096            .worker_id("activity-only-worker")
13097            .poll_timeout(Duration::from_millis(10));
13098
13099        worker.register_activity(
13100            "activity.only",
13101            |_ctx, _args| async move { Ok(Value::Null) },
13102        );
13103
13104        worker.run_until(async {}).await.expect("run worker");
13105    }
13106
13107    #[tokio::test]
13108    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
13109        let server = MockWorkerServer::start();
13110        let client = Client::builder(server.base_url())
13111            .timeout(Duration::from_secs(2))
13112            .build()
13113            .expect("client");
13114        let mut worker = Worker::new(client, "rust-workers")
13115            .worker_id("workflow-only-worker")
13116            .poll_timeout(Duration::from_millis(10));
13117
13118        worker.register_workflow(
13119            "workflow.only",
13120            |_ctx, _input| async move { Ok(Value::Null) },
13121        );
13122
13123        worker.run_until(async {}).await.expect("run worker");
13124    }
13125
13126    #[tokio::test]
13127    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
13128        let server = MockWorkerServer::start();
13129        let client = Client::builder(server.base_url())
13130            .timeout(Duration::from_secs(2))
13131            .build()
13132            .expect("client");
13133        let observations = Arc::new(Mutex::new(Vec::new()));
13134        let observed = Arc::clone(&observations);
13135        let mut worker = Worker::new(client, "rust-workers")
13136            .worker_id("observed-heartbeat-worker")
13137            .poll_timeout(Duration::from_millis(10))
13138            .on_worker_heartbeat(move |observation| {
13139                observed
13140                    .lock()
13141                    .expect("heartbeat observations")
13142                    .push(observation.clone());
13143            });
13144
13145        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
13146            Ok(Value::Null)
13147        });
13148        let acknowledged = Arc::clone(&observations);
13149        worker
13150            .run_until(async move {
13151                tokio::time::timeout(Duration::from_secs(2), async move {
13152                    loop {
13153                        if !acknowledged
13154                            .lock()
13155                            .expect("heartbeat observations")
13156                            .is_empty()
13157                        {
13158                            break;
13159                        }
13160                        tokio::time::sleep(Duration::from_millis(1)).await;
13161                    }
13162                })
13163                .await
13164                .expect("heartbeat acknowledgement within timeout");
13165            })
13166            .await
13167            .expect("run worker");
13168
13169        let observations = observations.lock().expect("heartbeat observations");
13170        let first = observations.first().expect("heartbeat acknowledgement");
13171        assert_eq!(first.worker_id, "observed-heartbeat-worker");
13172        assert_eq!(first.task_queue, "rust-workers");
13173        assert!(first.acknowledged_at_unix_millis > 0);
13174        assert_eq!(first.acknowledgement, json!({}));
13175    }
13176
13177    #[tokio::test]
13178    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
13179        let server = MockWorkerServer::delayed_heartbeat_worker();
13180        let client = Client::builder(server.base_url())
13181            .timeout(Duration::from_secs(3))
13182            .build()
13183            .expect("client");
13184        let observations = Arc::new(Mutex::new(Vec::new()));
13185        let observed = Arc::clone(&observations);
13186        let mut worker = Worker::new(client, "rust-snapshot-workers")
13187            .worker_id("rust-snapshot-worker")
13188            .poll_timeout(Duration::from_millis(10))
13189            .on_worker_heartbeat(move |observation| {
13190                observed
13191                    .lock()
13192                    .expect("heartbeat observations")
13193                    .push(observation.clone());
13194            });
13195
13196        worker.register_workflow("snapshot", |ctx, _input| async move {
13197            ctx.wait_signal("finish").await?;
13198            Ok(json!({"status": "finished"}))
13199        });
13200        worker.register_query("snapshot", "current", |ctx, _args| async move {
13201            Ok(json!(ctx
13202                .signals("increment")
13203                .iter()
13204                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13205                .sum::<i64>()))
13206        });
13207        worker.register_activity("cancel-aware", |_ctx, _args| async move {
13208            Ok(json!({"late": "completion"}))
13209        });
13210
13211        worker
13212            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
13213            .await
13214            .expect("delayed heartbeat must allow a clean worker shutdown");
13215
13216        let observations = observations.lock().expect("heartbeat observations");
13217        assert!(
13218            observations.len() >= 3,
13219            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
13220        );
13221        assert!(
13222            observations.windows(2).all(|pair| {
13223                pair[1].acknowledged_at_unix_millis
13224                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
13225                    >= 850
13226            }),
13227            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
13228        );
13229        drop(observations);
13230
13231        let heartbeat_times = server.request_times("/api/worker/heartbeat");
13232        let delayed_request_at = *heartbeat_times
13233            .get(1)
13234            .expect("intentionally delayed heartbeat request");
13235        let delay_window_start = delayed_request_at + Duration::from_millis(100);
13236        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
13237        for path in [
13238            "/api/worker/workflow-tasks/poll",
13239            "/api/worker/activity-tasks/poll",
13240            "/api/worker/query-tasks/poll",
13241        ] {
13242            assert!(
13243                server
13244                    .request_times(path)
13245                    .iter()
13246                    .any(|received_at| *received_at >= delay_window_start
13247                        && *received_at <= delay_window_end),
13248                "{path} must keep polling while a heartbeat acknowledgement is delayed"
13249            );
13250        }
13251        assert!(
13252            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
13253            "workflow work must be settled"
13254        );
13255        assert!(
13256            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
13257            "activity work must be settled"
13258        );
13259        assert!(
13260            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
13261            "query work must be settled"
13262        );
13263    }
13264
13265    #[tokio::test]
13266    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
13267        let server = MockWorkerServer::heartbeat_retry_worker();
13268        let client = Client::builder(server.base_url())
13269            .timeout(Duration::from_secs(2))
13270            .build()
13271            .expect("client");
13272        let observations = Arc::new(Mutex::new(Vec::new()));
13273        let observed = Arc::clone(&observations);
13274        let worker = Worker::new(client, "rust-workers")
13275            .worker_id("heartbeat-retry-worker")
13276            .retry_policy(WorkerRetryPolicy {
13277                max_retries: 1,
13278                initial_backoff: Duration::from_millis(300),
13279                max_backoff: Duration::from_millis(300),
13280            })
13281            .on_worker_heartbeat(move |observation| {
13282                observed
13283                    .lock()
13284                    .expect("heartbeat observations")
13285                    .push(observation.clone());
13286            });
13287
13288        worker
13289            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
13290            .await
13291            .expect("retryable heartbeat failure must remain bounded and recover");
13292
13293        let observations = observations.lock().expect("heartbeat observations");
13294        assert!(observations.len() >= 3, "heartbeat retry must recover");
13295        assert!(
13296            observations.windows(2).all(|pair| {
13297                pair[1]
13298                    .acknowledged_at_unix_millis
13299                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
13300                    >= 850
13301            }),
13302            "a successful retry must start a fresh advertised cadence: {observations:?}"
13303        );
13304        assert_eq!(
13305            server.request_count("/api/worker/heartbeat"),
13306            observations.len() + 1,
13307            "one retryable failure must add exactly one bounded request"
13308        );
13309    }
13310
13311    #[tokio::test]
13312    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
13313        let server = MockWorkerServer::waiting_query_worker();
13314        let client = Client::builder(server.base_url())
13315            .timeout(Duration::from_secs(2))
13316            .build()
13317            .expect("client");
13318        let observations = Arc::new(Mutex::new(Vec::new()));
13319        let observed = Arc::clone(&observations);
13320        let mut worker = Worker::new(client, "rust-snapshot-workers")
13321            .worker_id("rust-snapshot-worker")
13322            .poll_timeout(Duration::from_millis(10))
13323            .on_worker_heartbeat(move |observation| {
13324                observed
13325                    .lock()
13326                    .expect("heartbeat observations")
13327                    .push(observation.clone());
13328            });
13329
13330        worker.register_workflow("snapshot", |ctx, _input| async move {
13331            ctx.wait_signal("finish").await?;
13332            Ok(json!({"status": "finished"}))
13333        });
13334        worker.register_query("snapshot", "current", |ctx, _args| async move {
13335            let current = ctx
13336                .signals("increment")
13337                .iter()
13338                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13339                .sum::<i64>();
13340            Ok(json!(current))
13341        });
13342        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
13343
13344        worker
13345            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
13346            .await
13347            .expect("pending workflow and query poller must remain live until shutdown");
13348
13349        assert!(
13350            observations.lock().expect("heartbeat observations").len() >= 4,
13351            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
13352        );
13353        assert!(
13354            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
13355            "workflow polling must continue after empty replay acknowledgements"
13356        );
13357        assert!(
13358            server.request_count("/api/worker/query-tasks/poll") >= 2,
13359            "query polling must continue after serving the current query"
13360        );
13361        assert_eq!(
13362            server.request_body("/api/worker/register")["capabilities"],
13363            json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
13364        );
13365        assert_eq!(
13366            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
13367            json!({
13368                "queries": ["current"],
13369                "updates": ["replace"],
13370                "update_validators": [],
13371            })
13372        );
13373
13374        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
13375        assert_eq!(
13376            opened["commands"],
13377            json!([{
13378                "type": "open_signal_wait",
13379                "signal_name": "finish",
13380            }])
13381        );
13382
13383        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
13384            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
13385            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
13386            let failure = server.request_body(&fail_path);
13387            assert_eq!(
13388                failure["failure"]["type"],
13389                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
13390            );
13391            assert_eq!(server.request_count(&completion_path), 0);
13392        }
13393
13394        let query_completion =
13395            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
13396        assert_eq!(query_completion["result"], json!(8));
13397
13398        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
13399        assert_eq!(
13400            server.request_count(terminal_path),
13401            1,
13402            "the matching signal must settle the workflow exactly once"
13403        );
13404        let terminal = server.request_body(terminal_path);
13405        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
13406        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
13407        assert_eq!(
13408            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
13409                .expect("terminal workflow result"),
13410            json!({"status": "finished"})
13411        );
13412    }
13413
13414    #[tokio::test]
13415    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
13416        let server = MockWorkerServer::transient_worker_failures();
13417        let client = Client::builder(server.base_url())
13418            .timeout(Duration::from_secs(2))
13419            .build()
13420            .expect("client");
13421        let mut worker = Worker::new(client, "rust-workers")
13422            .worker_id("retry-worker")
13423            .poll_timeout(Duration::from_millis(10))
13424            .retry_policy(WorkerRetryPolicy {
13425                max_retries: 2,
13426                initial_backoff: Duration::from_millis(1),
13427                max_backoff: Duration::from_millis(1),
13428            });
13429        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13430        worker.register_activity(
13431            "counter.activity",
13432            |_ctx, _input| async move { Ok(Value::Null) },
13433        );
13434        worker.register_query(
13435            "counter",
13436            "current",
13437            |_ctx, _args| async move { Ok(json!(8)) },
13438        );
13439
13440        worker
13441            .run_until(tokio::time::sleep(Duration::from_millis(75)))
13442            .await
13443            .expect("transient failures must not stop the worker");
13444
13445        for path in [
13446            "/api/worker/heartbeat",
13447            "/api/worker/workflow-tasks/poll",
13448            "/api/worker/activity-tasks/poll",
13449            "/api/worker/query-tasks/poll",
13450        ] {
13451            assert!(
13452                server.request_count(path) >= 2,
13453                "{path} must continue after its transient failure"
13454            );
13455        }
13456    }
13457
13458    #[tokio::test]
13459    async fn worker_bounds_transport_retries() {
13460        let server = MockWorkerServer::unavailable_polls();
13461        let client = Client::builder(server.base_url())
13462            .timeout(Duration::from_secs(2))
13463            .build()
13464            .expect("client");
13465        let mut worker = Worker::new(client, "rust-workers")
13466            .worker_id("bounded-retry-worker")
13467            .poll_timeout(Duration::from_millis(10))
13468            .retry_policy(WorkerRetryPolicy {
13469                max_retries: 2,
13470                initial_backoff: Duration::from_millis(1),
13471                max_backoff: Duration::from_millis(1),
13472            });
13473        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13474
13475        let error = worker.run().await.expect_err("retry bound must terminate");
13476        assert!(matches!(error, Error::Transport(_)));
13477        assert_eq!(
13478            server.request_count("/api/worker/workflow-tasks/poll"),
13479            3,
13480            "one initial request plus exactly two retries"
13481        );
13482    }
13483
13484    #[tokio::test]
13485    async fn worker_retry_policy_can_disable_poll_retries() {
13486        let server = MockWorkerServer::unavailable_polls();
13487        let client = Client::builder(server.base_url())
13488            .timeout(Duration::from_secs(2))
13489            .build()
13490            .expect("client");
13491        let mut worker = Worker::new(client, "rust-workers")
13492            .worker_id("no-retry-worker")
13493            .poll_timeout(Duration::from_millis(10))
13494            .retry_policy(WorkerRetryPolicy {
13495                max_retries: 0,
13496                initial_backoff: Duration::from_millis(1),
13497                max_backoff: Duration::from_millis(1),
13498            });
13499        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13500
13501        let error = worker
13502            .run_once()
13503            .await
13504            .expect_err("disabled retries must return the first transport failure");
13505        assert!(matches!(error, Error::Transport(_)));
13506        assert_eq!(
13507            server.request_count("/api/worker/workflow-tasks/poll"),
13508            1,
13509            "max_retries=0 must send only the initial request"
13510        );
13511    }
13512
13513    #[tokio::test]
13514    async fn worker_does_not_retry_authentication_failures() {
13515        let server = MockWorkerServer::unauthorized_polls();
13516        let client = Client::builder(server.base_url())
13517            .timeout(Duration::from_secs(2))
13518            .build()
13519            .expect("client");
13520        let mut worker = Worker::new(client, "rust-workers")
13521            .worker_id("unauthorized-worker")
13522            .poll_timeout(Duration::from_millis(10));
13523        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13524
13525        let error = worker
13526            .run()
13527            .await
13528            .expect_err("authentication must terminate");
13529        let Error::Http { status, body } = error else {
13530            panic!("expected stable HTTP authentication error");
13531        };
13532        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13533        assert!(body.contains("authentication_failed"));
13534        assert_eq!(
13535            server.request_count("/api/worker/workflow-tasks/poll"),
13536            1,
13537            "authentication failures must not be retried"
13538        );
13539    }
13540
13541    #[derive(Clone, Debug)]
13542    struct CapturedRequest {
13543        method: String,
13544        path: String,
13545        authorization: Option<String>,
13546        namespace: Option<String>,
13547        worker_protocol: Option<String>,
13548        control_protocol: Option<String>,
13549        body: String,
13550        received_at: Instant,
13551    }
13552
13553    struct MockWorkerServer {
13554        addr: SocketAddr,
13555        stop: Arc<AtomicBool>,
13556        requests: Arc<Mutex<Vec<CapturedRequest>>>,
13557        thread: Option<thread::JoinHandle<()>>,
13558    }
13559
13560    #[derive(Clone, Copy, Default)]
13561    struct MockWorkerBehavior {
13562        reject_query_protocol: bool,
13563        reject_query_completion: bool,
13564        waiting_query_worker: bool,
13565        decline_registration: bool,
13566        complete_named_signal: bool,
13567        poll_failures_per_path: usize,
13568        heartbeat_failures: usize,
13569        heartbeat_failure_request: Option<usize>,
13570        delayed_heartbeat_request: Option<usize>,
13571        heartbeat_response_delay: Duration,
13572        concurrent_requests: bool,
13573        unauthorized_polls: bool,
13574        reject_registration: bool,
13575        reject_deregistration: bool,
13576        reject_deregistration_protocol: bool,
13577        cancelled_activity: bool,
13578        draining_polls: bool,
13579        workflow_completion_status: Option<&'static str>,
13580        workflow_completion_body: Option<&'static str>,
13581    }
13582
13583    impl MockWorkerServer {
13584        fn start() -> Self {
13585            Self::start_with_behavior(MockWorkerBehavior::default())
13586        }
13587
13588        fn reject_query_protocol() -> Self {
13589            Self::start_with_behavior(MockWorkerBehavior {
13590                reject_query_protocol: true,
13591                ..MockWorkerBehavior::default()
13592            })
13593        }
13594
13595        fn reject_query_completion() -> Self {
13596            Self::start_with_behavior(MockWorkerBehavior {
13597                reject_query_completion: true,
13598                ..MockWorkerBehavior::default()
13599            })
13600        }
13601
13602        fn waiting_query_worker() -> Self {
13603            Self::start_with_behavior(MockWorkerBehavior {
13604                waiting_query_worker: true,
13605                complete_named_signal: true,
13606                ..MockWorkerBehavior::default()
13607            })
13608        }
13609
13610        fn transient_worker_failures() -> Self {
13611            Self::start_with_behavior(MockWorkerBehavior {
13612                poll_failures_per_path: 1,
13613                heartbeat_failures: 1,
13614                ..MockWorkerBehavior::default()
13615            })
13616        }
13617
13618        fn consecutive_poll_failures(count: usize) -> Self {
13619            Self::start_with_behavior(MockWorkerBehavior {
13620                poll_failures_per_path: count,
13621                ..MockWorkerBehavior::default()
13622            })
13623        }
13624
13625        fn delayed_heartbeat_worker() -> Self {
13626            Self::start_with_behavior(MockWorkerBehavior {
13627                waiting_query_worker: true,
13628                delayed_heartbeat_request: Some(2),
13629                heartbeat_response_delay: Duration::from_millis(1_500),
13630                concurrent_requests: true,
13631                cancelled_activity: true,
13632                ..MockWorkerBehavior::default()
13633            })
13634        }
13635
13636        fn heartbeat_retry_worker() -> Self {
13637            Self::start_with_behavior(MockWorkerBehavior {
13638                waiting_query_worker: true,
13639                heartbeat_failure_request: Some(2),
13640                concurrent_requests: true,
13641                ..MockWorkerBehavior::default()
13642            })
13643        }
13644
13645        fn unavailable_polls() -> Self {
13646            Self::start_with_behavior(MockWorkerBehavior {
13647                poll_failures_per_path: usize::MAX,
13648                ..MockWorkerBehavior::default()
13649            })
13650        }
13651
13652        fn unauthorized_polls() -> Self {
13653            Self::start_with_behavior(MockWorkerBehavior {
13654                unauthorized_polls: true,
13655                ..MockWorkerBehavior::default()
13656            })
13657        }
13658
13659        fn rejected_registration() -> Self {
13660            Self::start_with_behavior(MockWorkerBehavior {
13661                reject_registration: true,
13662                ..MockWorkerBehavior::default()
13663            })
13664        }
13665
13666        fn declined_registration() -> Self {
13667            Self::start_with_behavior(MockWorkerBehavior {
13668                decline_registration: true,
13669                ..MockWorkerBehavior::default()
13670            })
13671        }
13672
13673        fn rejected_deregistration() -> Self {
13674            Self::start_with_behavior(MockWorkerBehavior {
13675                reject_deregistration: true,
13676                ..MockWorkerBehavior::default()
13677            })
13678        }
13679
13680        fn rejected_deregistration_protocol() -> Self {
13681            Self::start_with_behavior(MockWorkerBehavior {
13682                reject_deregistration_protocol: true,
13683                ..MockWorkerBehavior::default()
13684            })
13685        }
13686
13687        fn unauthorized_polls_and_rejected_deregistration() -> Self {
13688            Self::start_with_behavior(MockWorkerBehavior {
13689                unauthorized_polls: true,
13690                reject_deregistration: true,
13691                ..MockWorkerBehavior::default()
13692            })
13693        }
13694
13695        fn cancelled_activity() -> Self {
13696            Self::start_with_behavior(MockWorkerBehavior {
13697                cancelled_activity: true,
13698                ..MockWorkerBehavior::default()
13699            })
13700        }
13701
13702        fn draining_polls() -> Self {
13703            Self::start_with_behavior(MockWorkerBehavior {
13704                draining_polls: true,
13705                ..MockWorkerBehavior::default()
13706            })
13707        }
13708
13709        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13710            Self::start_with_behavior(MockWorkerBehavior {
13711                workflow_completion_status: Some(status),
13712                workflow_completion_body: Some(body),
13713                ..MockWorkerBehavior::default()
13714            })
13715        }
13716
13717        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13718            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13719            listener
13720                .set_nonblocking(true)
13721                .expect("configure mock listener");
13722            let addr = listener.local_addr().expect("mock server address");
13723            let stop = Arc::new(AtomicBool::new(false));
13724            let server_stop = Arc::clone(&stop);
13725            let requests = Arc::new(Mutex::new(Vec::new()));
13726            let server_requests = Arc::clone(&requests);
13727            let thread = thread::spawn(move || {
13728                let mut request_threads = Vec::new();
13729                while !server_stop.load(Ordering::SeqCst) {
13730                    match listener.accept() {
13731                        Ok((mut stream, _)) => {
13732                            if behavior.concurrent_requests {
13733                                let requests = Arc::clone(&server_requests);
13734                                request_threads.push(thread::spawn(move || {
13735                                    handle_mock_worker_request(&mut stream, &requests, behavior)
13736                                }));
13737                            } else {
13738                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
13739                            }
13740                        }
13741                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13742                            let mut index = 0;
13743                            while index < request_threads.len() {
13744                                if request_threads[index].is_finished() {
13745                                    request_threads
13746                                        .swap_remove(index)
13747                                        .join()
13748                                        .expect("join mock request");
13749                                } else {
13750                                    index += 1;
13751                                }
13752                            }
13753                            thread::sleep(Duration::from_millis(5));
13754                        }
13755                        Err(_) => break,
13756                    }
13757                }
13758                for request_thread in request_threads {
13759                    request_thread.join().expect("join mock request");
13760                }
13761            });
13762
13763            Self {
13764                addr,
13765                stop,
13766                requests,
13767                thread: Some(thread),
13768            }
13769        }
13770
13771        fn base_url(&self) -> String {
13772            format!("http://{}", self.addr)
13773        }
13774
13775        fn worker_protocol_for(&self, path: &str) -> Option<String> {
13776            self.requests
13777                .lock()
13778                .expect("captured requests")
13779                .iter()
13780                .find(|request| request.path == path)
13781                .and_then(|request| request.worker_protocol.clone())
13782        }
13783
13784        fn control_protocol_for(&self, path: &str) -> Option<String> {
13785            self.requests
13786                .lock()
13787                .expect("captured requests")
13788                .iter()
13789                .find(|request| request.path == path)
13790                .and_then(|request| request.control_protocol.clone())
13791        }
13792
13793        fn method_for(&self, path: &str) -> Option<String> {
13794            self.requests
13795                .lock()
13796                .expect("captured requests")
13797                .iter()
13798                .find(|request| request.path == path)
13799                .map(|request| request.method.clone())
13800        }
13801
13802        fn authorization_for(&self, path: &str) -> Option<String> {
13803            self.requests
13804                .lock()
13805                .expect("captured requests")
13806                .iter()
13807                .find(|request| request.path == path)
13808                .and_then(|request| request.authorization.clone())
13809        }
13810
13811        fn namespace_for(&self, path: &str) -> Option<String> {
13812            self.requests
13813                .lock()
13814                .expect("captured requests")
13815                .iter()
13816                .find(|request| request.path == path)
13817                .and_then(|request| request.namespace.clone())
13818        }
13819
13820        fn request_count(&self, path: &str) -> usize {
13821            self.requests
13822                .lock()
13823                .expect("captured requests")
13824                .iter()
13825                .filter(|request| request.path == path)
13826                .count()
13827        }
13828
13829        fn captured_paths(&self) -> Vec<String> {
13830            self.requests
13831                .lock()
13832                .expect("captured requests")
13833                .iter()
13834                .map(|request| request.path.clone())
13835                .collect()
13836        }
13837
13838        fn request_times(&self, path: &str) -> Vec<Instant> {
13839            self.requests
13840                .lock()
13841                .expect("captured requests")
13842                .iter()
13843                .filter(|request| request.path == path)
13844                .map(|request| request.received_at)
13845                .collect()
13846        }
13847
13848        fn request_body(&self, path: &str) -> Value {
13849            let requests = self.requests.lock().expect("captured requests");
13850            let body = &requests
13851                .iter()
13852                .find(|request| request.path == path)
13853                .unwrap_or_else(|| panic!("missing request for {path}"))
13854                .body;
13855            serde_json::from_str(body).unwrap_or_else(|error| {
13856                panic!("invalid JSON request body for {path}: {error}: {body:?}")
13857            })
13858        }
13859
13860        fn request_bodies(&self, path: &str) -> Vec<Value> {
13861            self.requests
13862                .lock()
13863                .expect("captured requests")
13864                .iter()
13865                .filter(|request| request.path == path)
13866                .map(|request| {
13867                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
13868                        panic!(
13869                            "invalid JSON request body for {path}: {error}: {:?}",
13870                            request.body
13871                        )
13872                    })
13873                })
13874                .collect()
13875        }
13876    }
13877
13878    impl Drop for MockWorkerServer {
13879        fn drop(&mut self) {
13880            self.stop.store(true, Ordering::SeqCst);
13881            let _ = TcpStream::connect(self.addr);
13882
13883            if let Some(thread) = self.thread.take() {
13884                thread.join().expect("join mock server");
13885            }
13886        }
13887    }
13888
13889    fn handle_mock_worker_request(
13890        stream: &mut TcpStream,
13891        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13892        behavior: MockWorkerBehavior,
13893    ) {
13894        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13895        let mut buffer = [0_u8; 8192];
13896        let mut request = Vec::new();
13897
13898        loop {
13899            match stream.read(&mut buffer) {
13900                Ok(0) => break,
13901                Ok(read) => {
13902                    request.extend_from_slice(&buffer[..read]);
13903                    if mock_request_is_complete(&request) {
13904                        break;
13905                    }
13906                }
13907                Err(error)
13908                    if matches!(
13909                        error.kind(),
13910                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
13911                    ) =>
13912                {
13913                    break;
13914                }
13915                Err(_) => return,
13916            }
13917        }
13918
13919        let request = String::from_utf8_lossy(&request);
13920        let body = request
13921            .split_once("\r\n\r\n")
13922            .map(|(_, body)| body)
13923            .unwrap_or_default();
13924        let path = request
13925            .lines()
13926            .next()
13927            .and_then(|line| line.split_whitespace().nth(1))
13928            .unwrap_or_default();
13929        let method = request
13930            .lines()
13931            .next()
13932            .and_then(|line| line.split_whitespace().next())
13933            .unwrap_or_default();
13934        let authorization = request.lines().find_map(|line| {
13935            let (name, value) = line.split_once(':')?;
13936            name.eq_ignore_ascii_case("Authorization")
13937                .then(|| value.trim().to_string())
13938        });
13939        let namespace = request.lines().find_map(|line| {
13940            let (name, value) = line.split_once(':')?;
13941            name.eq_ignore_ascii_case("X-Namespace")
13942                .then(|| value.trim().to_string())
13943        });
13944        let worker_protocol = request.lines().find_map(|line| {
13945            let (name, value) = line.split_once(':')?;
13946            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
13947                .then(|| value.trim().to_string())
13948        });
13949        let control_protocol = request.lines().find_map(|line| {
13950            let (name, value) = line.split_once(':')?;
13951            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
13952                .then(|| value.trim().to_string())
13953        });
13954        let request_number = {
13955            let mut requests = requests.lock().expect("captured requests");
13956            requests.push(CapturedRequest {
13957                method: method.to_string(),
13958                path: path.to_string(),
13959                authorization,
13960                namespace,
13961                worker_protocol: worker_protocol.clone(),
13962                control_protocol,
13963                body: body.to_string(),
13964                received_at: Instant::now(),
13965            });
13966            requests
13967                .iter()
13968                .filter(|request| request.path == path)
13969                .count()
13970        };
13971
13972        if behavior.reject_registration && path == "/api/worker/register" {
13973            write_mock_response(
13974                stream,
13975                "503 Service Unavailable",
13976                r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
13977            );
13978            return;
13979        }
13980
13981        if path.starts_with("/api/worker/registrations/") {
13982            if behavior.reject_deregistration_protocol {
13983                write_mock_response(
13984                    stream,
13985                    "400 Bad Request",
13986                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
13987                );
13988            } else if behavior.reject_deregistration {
13989                write_mock_response(
13990                    stream,
13991                    "403 Forbidden",
13992                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
13993                );
13994            } else {
13995                write_mock_response(
13996                    stream,
13997                    "200 OK",
13998                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
13999                );
14000            }
14001            return;
14002        }
14003
14004        let is_poll = matches!(
14005            path,
14006            "/api/worker/workflow-tasks/poll"
14007                | "/api/worker/activity-tasks/poll"
14008                | "/api/worker/query-tasks/poll"
14009        );
14010        if is_poll && request_number <= behavior.poll_failures_per_path {
14011            return;
14012        }
14013        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14014            return;
14015        }
14016        if path == "/api/worker/heartbeat"
14017            && behavior.heartbeat_failure_request == Some(request_number)
14018        {
14019            return;
14020        }
14021        if path == "/api/worker/heartbeat"
14022            && behavior.delayed_heartbeat_request == Some(request_number)
14023        {
14024            thread::sleep(behavior.heartbeat_response_delay);
14025        }
14026        if behavior.unauthorized_polls && is_poll {
14027            write_mock_response(
14028                stream,
14029                "401 Unauthorized",
14030                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14031            );
14032            return;
14033        }
14034        if behavior.draining_polls && is_poll {
14035            write_mock_response(
14036                stream,
14037                "409 Conflict",
14038                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14039            );
14040            return;
14041        }
14042
14043        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
14044            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
14045            let body = format!(
14046                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
14047            );
14048            write_mock_response(stream, "400 Bad Request", &body);
14049            return;
14050        }
14051
14052        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
14053        {
14054            write_mock_response(
14055                stream,
14056                "409 Conflict",
14057                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
14058            );
14059            return;
14060        }
14061
14062        if behavior.workflow_completion_status.is_some()
14063            && path == "/api/worker/workflow-tasks/poll"
14064            && request_number == 1
14065        {
14066            write_mock_response(
14067                stream,
14068                "200 OK",
14069                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
14070            );
14071            return;
14072        }
14073
14074        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
14075            if let (Some(status), Some(body)) = (
14076                behavior.workflow_completion_status,
14077                behavior.workflow_completion_body,
14078            ) {
14079                write_mock_response(stream, status, body);
14080                return;
14081            }
14082        }
14083
14084        if behavior.waiting_query_worker {
14085            if behavior.complete_named_signal
14086                && path == "/api/worker/workflow-tasks/poll"
14087                && request_number == 1
14088            {
14089                let body = json!({
14090                    "task": {
14091                        "task_id": "snapshot-open",
14092                        "workflow_id": "snapshot-1",
14093                        "run_id": "snapshot-run-1",
14094                        "workflow_type": "snapshot",
14095                        "payload_codec": DEFAULT_CODEC,
14096                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14097                            .expect("Avro workflow arguments"),
14098                        "history_events": [],
14099                        "workflow_task_attempt": 1,
14100                        "lease_owner": "rust-snapshot-worker"
14101                    }
14102                })
14103                .to_string();
14104                write_mock_response(stream, "200 OK", &body);
14105                return;
14106            }
14107
14108            let signal_request = request_number - usize::from(behavior.complete_named_signal);
14109            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
14110            if path == "/api/worker/workflow-tasks/poll"
14111                && signal_request >= 1
14112                && signal_request <= signal_request_limit
14113            {
14114                let finish = behavior.complete_named_signal && signal_request == 3;
14115                let amounts = if signal_request == 1 {
14116                    vec![3]
14117                } else {
14118                    vec![3, 5]
14119                };
14120                let task_id = if signal_request == 1 {
14121                    "snapshot-wait-3"
14122                } else if finish {
14123                    "snapshot-finish"
14124                } else {
14125                    "snapshot-wait-5"
14126                };
14127                let mut history_events = std::iter::once(json!({
14128                    "event_type": "SignalWaitOpened",
14129                    "payload": {"sequence": 1, "signal_name": "finish"}
14130                }))
14131                .chain(amounts.iter().enumerate().map(|(index, amount)| {
14132                    json!({
14133                        "event_type": "SignalReceived",
14134                        "payload": {
14135                            "signal_id": format!("increment-{amount}"),
14136                            "signal_name": "increment",
14137                            "workflow_sequence": index + 2,
14138                            "payload_codec": DEFAULT_CODEC,
14139                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14140                                .expect("Avro signal envelope")
14141                        }
14142                    })
14143                }))
14144                .collect::<Vec<_>>();
14145                let (resume_id, resume_name, resume_arguments) = if finish {
14146                    history_events.push(json!({
14147                        "event_type": "SignalReceived",
14148                        "payload": {
14149                            "signal_id": "finish",
14150                            "signal_name": "finish",
14151                            "workflow_sequence": 4,
14152                            "payload_codec": DEFAULT_CODEC,
14153                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14154                                .expect("Avro finish signal envelope")
14155                        }
14156                    }));
14157                    (
14158                        "finish".to_string(),
14159                        "finish".to_string(),
14160                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
14161                            .expect("Avro finish resume signal"),
14162                    )
14163                } else {
14164                    let amount = amounts.last().expect("amount");
14165                    (
14166                        format!("increment-{amount}"),
14167                        "increment".to_string(),
14168                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14169                            .expect("Avro increment resume signal"),
14170                    )
14171                };
14172                let body = json!({
14173                    "task": {
14174                        "task_id": task_id,
14175                        "workflow_id": "snapshot-1",
14176                        "run_id": "snapshot-run-1",
14177                        "workflow_type": "snapshot",
14178                        "payload_codec": DEFAULT_CODEC,
14179                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14180                            .expect("Avro workflow arguments"),
14181                        "history_events": history_events,
14182                        "workflow_task_attempt": 1,
14183                        "workflow_signal_id": resume_id,
14184                        "signal_name": resume_name,
14185                        "signal_arguments": resume_arguments,
14186                        "lease_owner": "rust-snapshot-worker"
14187                    }
14188                })
14189                .to_string();
14190                write_mock_response(stream, "200 OK", &body);
14191                return;
14192            }
14193
14194            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
14195                let history_events = [3, 5]
14196                    .into_iter()
14197                    .enumerate()
14198                    .map(|(index, amount)| {
14199                        json!({
14200                            "event_type": "SignalReceived",
14201                            "payload": {
14202                                "signal_id": format!("increment-{amount}"),
14203                                "signal_name": "increment",
14204                                "workflow_sequence": index + 2,
14205                                "payload_codec": DEFAULT_CODEC,
14206                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14207                                    .expect("Avro query signal envelope")
14208                            }
14209                        })
14210                    })
14211                    .collect::<Vec<_>>();
14212                let body = json!({
14213                    "task": {
14214                        "query_task_id": "snapshot-current",
14215                        "query_task_attempt": 1,
14216                        "lease_owner": "rust-snapshot-worker",
14217                        "workflow_id": "snapshot-1",
14218                        "run_id": "snapshot-run-1",
14219                        "workflow_type": "snapshot",
14220                        "query_name": "current",
14221                        "payload_codec": DEFAULT_CODEC,
14222                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14223                            .expect("Avro workflow arguments"),
14224                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14225                            .expect("Avro query arguments"),
14226                        "history_events": history_events,
14227                        "run_status": "waiting"
14228                    }
14229                })
14230                .to_string();
14231                write_mock_response(stream, "200 OK", &body);
14232                return;
14233            }
14234
14235            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
14236                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
14237            {
14238                write_mock_response(
14239                    stream,
14240                    "200 OK",
14241                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
14242                );
14243                return;
14244            }
14245
14246            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
14247                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
14248                return;
14249            }
14250
14251            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
14252                write_mock_response(
14253                    stream,
14254                    "200 OK",
14255                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
14256                );
14257                return;
14258            }
14259
14260            if path == "/api/worker/query-tasks/snapshot-current/complete" {
14261                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
14262                return;
14263            }
14264        }
14265
14266        if matches!(
14267            path,
14268            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
14269        ) {
14270            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14271                .expect("typed mock result");
14272            let body = json!({
14273                "result": typed_fidelity_probe().into_json().expect("result projection"),
14274                "result_envelope": result,
14275            })
14276            .to_string();
14277            write_mock_response(stream, "200 OK", &body);
14278            return;
14279        }
14280
14281        if path == "/api/workflows/typed-1" {
14282            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14283                .expect("typed mock result");
14284            let body = json!({
14285                "workflow_id": "typed-1",
14286                "run_id": "run-typed-1",
14287                "workflow_type": "typed.echo",
14288                "status": "completed",
14289                "output": typed_fidelity_probe().into_json().expect("output projection"),
14290                "output_envelope": result,
14291            })
14292            .to_string();
14293            write_mock_response(stream, "200 OK", &body);
14294            return;
14295        }
14296
14297        let (status, body) = match path {
14298            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
14299            "/api/workflows" => (
14300                "201 Created",
14301                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
14302            ),
14303            "/api/worker/register" if behavior.decline_registration => (
14304                "200 OK",
14305                r#"{"worker_id":"declined-worker","registered":false}"#,
14306            ),
14307            "/api/worker/register" if behavior.waiting_query_worker => (
14308                "200 OK",
14309                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
14310            ),
14311            "/api/worker/register" => (
14312                "200 OK",
14313                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
14314            ),
14315            "/api/worker/heartbeat" => ("200 OK", "{}"),
14316            "/api/worker/activity-tasks/poll"
14317                if behavior.cancelled_activity && request_number == 1 =>
14318            {
14319                (
14320                    "200 OK",
14321                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
14322                )
14323            }
14324            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
14325                ("200 OK", r#"{"task":null}"#)
14326            }
14327            "/api/worker/query-tasks/poll"
14328                if behavior.reject_query_completion && request_number == 1 =>
14329            {
14330                (
14331                    "200 OK",
14332                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
14333                )
14334            }
14335            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
14336            "/api/worker/query-tasks/query-capture/complete"
14337            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
14338            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
14339                "200 OK",
14340                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
14341            ),
14342            "/api/worker/activity-tasks/activity-cancel/complete" => (
14343                "409 Conflict",
14344                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
14345            ),
14346            "/api/worker/activity-tasks/activity-typed/complete"
14347            | "/api/worker/activity-tasks/activity-typed/fail"
14348            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
14349            "/api/workflows/counter-1/query/current" => (
14350                "200 OK",
14351                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
14352            ),
14353            "/api/workflows/counter-1/query/missing" => (
14354                "404 Not Found",
14355                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
14356            ),
14357            "/api/workflows/wf-lifecycle/cancel" => (
14358                "200 OK",
14359                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
14360            ),
14361            "/api/workflows/wf-lifecycle/terminate" => (
14362                "200 OK",
14363                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
14364            ),
14365            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
14366                "200 OK",
14367                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
14368            ),
14369            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
14370                "200 OK",
14371                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
14372            ),
14373            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
14374            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
14375                "409 Conflict",
14376                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
14377            ),
14378            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
14379                "200 OK",
14380                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
14381            ),
14382            "/api/workflows/wf-cancelled" => (
14383                "200 OK",
14384                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
14385            ),
14386            "/api/workflows/wf-terminated" => (
14387                "200 OK",
14388                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
14389            ),
14390            "/api/workflows/wf-timed-out" => (
14391                "200 OK",
14392                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
14393            ),
14394            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
14395                "200 OK",
14396                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
14397            ),
14398            "/api/workflows/wf-selected" => (
14399                "200 OK",
14400                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
14401            ),
14402            "/api/workflows/wf-selected/runs/run-selected" => (
14403                "200 OK",
14404                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
14405            ),
14406            _ => ("404 Not Found", r#"{"message":"not found"}"#),
14407        };
14408        write_mock_response(stream, status, body);
14409    }
14410
14411    fn mock_request_is_complete(request: &[u8]) -> bool {
14412        let Some(header_end) = request
14413            .windows(4)
14414            .position(|window| window == b"\r\n\r\n")
14415            .map(|position| position + 4)
14416        else {
14417            return false;
14418        };
14419        let headers = String::from_utf8_lossy(&request[..header_end]);
14420        let content_length = headers.lines().find_map(|line| {
14421            let (name, value) = line.split_once(':')?;
14422            name.eq_ignore_ascii_case("content-length")
14423                .then(|| value.trim().parse::<usize>().ok())
14424                .flatten()
14425        });
14426
14427        request.len() >= header_end + content_length.unwrap_or(0)
14428    }
14429
14430    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
14431        let response = format!(
14432            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
14433            body.len()
14434        );
14435
14436        let _ = stream.write_all(response.as_bytes());
14437        let _ = stream.flush();
14438    }
14439}