Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21    de::DeserializeOwned,
22    ser::{SerializeMap, SerializeSeq},
23    Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34/// Worker-registration capability for server-routed read-only queries.
35pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36/// Worker-registration capability for synchronous workflow updates.
37pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38/// First additive worker protocol that defines query-task transport.
39pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43    "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47    "lease_expired",
48    "query_task_not_found",
49    "query_task_not_leased",
50    "query_task_timed_out",
51];
52
53/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
54pub const AVRO_VALUE_SCHEMA_JSON: &str =
55    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64    ControlPlane,
65    Worker(&'static str),
66}
67
68pub type Result<T> = std::result::Result<T, Error>;
69
70#[derive(Debug, Error)]
71pub enum Error {
72    #[error("transport error: {0}")]
73    Transport(#[from] reqwest::Error),
74    #[error(
75        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
76    )]
77    InvalidBaseUrl,
78    #[error("json error: {0}")]
79    Json(#[from] serde_json::Error),
80    #[error("http {status}: {body}")]
81    Http {
82        status: reqwest::StatusCode,
83        body: String,
84    },
85    #[error("codec error: {0}")]
86    Codec(String),
87    #[error(transparent)]
88    QueryFailed(QueryFailure),
89    #[error(transparent)]
90    Protocol(ProtocolFailure),
91    #[error(transparent)]
92    NonDeterministicReplay(ReplayFailure),
93    #[error(transparent)]
94    ChildWorkflowFailed(ChildWorkflowFailure),
95    #[error(transparent)]
96    ActivityFailed(ActivityFailure),
97    #[error(transparent)]
98    WorkflowCommandRejected(WorkflowCommandRejection),
99    #[error(transparent)]
100    WorkflowFailed(WorkflowTerminalOutcome),
101    #[error(transparent)]
102    WorkflowCancelled(WorkflowTerminalOutcome),
103    #[error(transparent)]
104    WorkflowTerminated(WorkflowTerminalOutcome),
105    #[error(transparent)]
106    WorkflowTimedOut(WorkflowTerminalOutcome),
107    #[error(transparent)]
108    ActivityTaskRejected(ActivityTaskRejection),
109    #[error("workflow handler {0:?} is not registered")]
110    WorkflowNotRegistered(String),
111    #[error("activity handler {0:?} is not registered")]
112    ActivityNotRegistered(String),
113    #[error("workflow future yielded without emitting a durable command")]
114    WorkflowYieldedWithoutCommand,
115    #[error("workflow state lock is poisoned")]
116    WorkflowStatePoisoned,
117    #[error("timer duration is too large for the worker protocol")]
118    TimerDurationOverflow,
119    #[error("operation timed out")]
120    Timeout,
121    #[error(
122        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
123    )]
124    MissingRoleCredentials {
125        role: &'static str,
126        opposite_role: &'static str,
127    },
128    #[error("worker loop error: {0}")]
129    WorkerLoop(String),
130    #[error("{primary}; worker deregistration also failed: {deregistration}")]
131    WorkerShutdown {
132        primary: Box<Error>,
133        deregistration: Box<Error>,
134    },
135    #[error("invalid child workflow options: {0}")]
136    InvalidChildWorkflowOptions(String),
137    #[error(transparent)]
138    InvalidActivityOptions(ActivityOptionsError),
139    #[error(transparent)]
140    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
141    #[doc(hidden)]
142    #[error("workflow requested continue as new")]
143    ContinueAsNew(ContinueAsNewRequest),
144}
145
146/// The lifecycle command sent to a workflow execution.
147#[derive(Clone, Copy, Debug, PartialEq, Eq)]
148pub enum WorkflowCommandKind {
149    Cancel,
150    Terminate,
151}
152
153impl WorkflowCommandKind {
154    fn as_str(self) -> &'static str {
155        match self {
156            Self::Cancel => "cancel",
157            Self::Terminate => "terminate",
158        }
159    }
160}
161
162/// Optional structured fields for a cancellation or termination request.
163#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
164pub struct WorkflowCommandOptions {
165    #[serde(skip_serializing_if = "Option::is_none")]
166    pub reason: Option<String>,
167    #[serde(skip_serializing_if = "Option::is_none")]
168    pub request_id: Option<String>,
169}
170
171/// Server-enforced timeout policy for a workflow start.
172///
173/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
174/// only bounds how long the caller waits. A server deadline produces a terminal
175/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
176/// `run_timeout`.
177#[derive(Clone, Debug, PartialEq, Eq)]
178pub struct WorkflowStartOptions {
179    pub execution_timeout_seconds: u64,
180    pub run_timeout_seconds: u64,
181}
182
183impl Default for WorkflowStartOptions {
184    fn default() -> Self {
185        Self {
186            execution_timeout_seconds: 3600,
187            run_timeout_seconds: 600,
188        }
189    }
190}
191
192impl WorkflowStartOptions {
193    pub fn new() -> Self {
194        Self::default()
195    }
196
197    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
198        self.execution_timeout_seconds = seconds;
199        self
200    }
201
202    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
203        self.run_timeout_seconds = seconds;
204        self
205    }
206
207    fn validate(&self) -> Result<()> {
208        if self.execution_timeout_seconds == 0 {
209            return Err(Error::Codec(
210                "execution_timeout_seconds must be at least 1".to_string(),
211            ));
212        }
213        if self.run_timeout_seconds == 0 {
214            return Err(Error::Codec(
215                "run_timeout_seconds must be at least 1".to_string(),
216            ));
217        }
218        if self.run_timeout_seconds > self.execution_timeout_seconds {
219            return Err(Error::Codec(
220                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
221            ));
222        }
223
224        Ok(())
225    }
226}
227
228/// Optional routing overrides for a continue-as-new transition.
229///
230/// Omitted values retain the current workflow type and task queue. Server-owned
231/// instance metadata is not accepted here and is carried by the server.
232#[derive(Clone, Debug, Default, PartialEq, Eq)]
233pub struct ContinueAsNewOptions {
234    pub workflow_type: Option<String>,
235    pub task_queue: Option<String>,
236}
237
238impl ContinueAsNewOptions {
239    pub fn new() -> Self {
240        Self::default()
241    }
242
243    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
244        self.workflow_type = Some(workflow_type.into());
245        self
246    }
247
248    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
249        self.task_queue = Some(task_queue.into());
250        self
251    }
252
253    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
254        for (field, value) in [
255            ("workflow_type", self.workflow_type.as_deref()),
256            ("task_queue", self.task_queue.as_deref()),
257        ] {
258            if value.is_some_and(|value| value.trim().is_empty()) {
259                return Err(ContinueAsNewOptionsError {
260                    field,
261                    message: format!("{field} must not be empty"),
262                });
263            }
264        }
265        Ok(())
266    }
267}
268
269/// A stable validation error raised before a continue-as-new command is emitted.
270#[derive(Clone, Debug, Error, PartialEq, Eq)]
271#[error("invalid continue-as-new option {field}: {message}")]
272pub struct ContinueAsNewOptionsError {
273    pub field: &'static str,
274    pub message: String,
275}
276
277/// Public history-budget information attached to the current workflow task.
278#[derive(Clone, Debug, Default, PartialEq, Eq)]
279pub struct WorkflowHistoryBudget {
280    pub event_count: u64,
281    pub size_bytes: Option<u64>,
282    pub continue_as_new_recommended: bool,
283    pub pressure: Option<String>,
284}
285
286#[doc(hidden)]
287#[derive(Clone, Debug)]
288pub struct ContinueAsNewRequest {
289    arguments: AvroValue,
290    options: ContinueAsNewOptions,
291}
292
293impl WorkflowCommandOptions {
294    pub fn new() -> Self {
295        Self::default()
296    }
297
298    pub fn reason(mut self, reason: impl Into<String>) -> Self {
299        self.reason = Some(reason.into());
300        self
301    }
302
303    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
304        self.request_id = Some(request_id.into());
305        self
306    }
307}
308
309/// The accepted, machine-readable result of a lifecycle command.
310#[derive(Clone, Debug, PartialEq)]
311pub struct WorkflowCommandResult {
312    pub command: WorkflowCommandKind,
313    pub workflow_id: String,
314    pub run_id: Option<String>,
315    pub outcome: Option<String>,
316    pub reason: Option<String>,
317    pub command_status: Option<String>,
318    pub raw: Value,
319}
320
321/// A stable rejection returned by instance- or selected-run lifecycle commands.
322#[derive(Clone, Debug, Error)]
323#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
324pub struct WorkflowCommandRejection {
325    pub command: WorkflowCommandKind,
326    pub status: u16,
327    pub reason: String,
328    pub message: String,
329    pub workflow_id: String,
330    pub run_id: Option<String>,
331    pub target_scope: Option<String>,
332    pub body: Value,
333}
334
335/// Stable terminal categories returned by [`WorkflowHandle::result`].
336#[derive(Clone, Copy, Debug, PartialEq, Eq)]
337pub enum WorkflowTerminalKind {
338    Failed,
339    Cancelled,
340    Terminated,
341    TimedOut,
342}
343
344/// A typed terminal workflow outcome with durable identity and failure metadata.
345///
346/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
347/// of parsing its display representation. Fields remain `None` when an older
348/// server did not publish that metadata.
349#[derive(Clone, Debug, Error)]
350#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
351pub struct WorkflowTerminalOutcome {
352    pub kind: WorkflowTerminalKind,
353    pub workflow_id: String,
354    pub run_id: Option<String>,
355    pub reason: String,
356    pub failure_category: Option<String>,
357    pub failure_id: Option<String>,
358    pub exception_type: Option<String>,
359    pub exception_class: Option<String>,
360    pub non_retryable: Option<bool>,
361    pub message: Option<String>,
362    pub exception: Option<Value>,
363    pub raw: Value,
364}
365
366/// A worker-side activity settlement or heartbeat rejected by durable state.
367#[derive(Clone, Debug, Error)]
368#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
369pub struct ActivityTaskRejection {
370    pub operation: String,
371    pub status: u16,
372    pub reason: String,
373    pub task_id: String,
374    pub activity_attempt_id: String,
375    pub cancel_requested: bool,
376    pub can_continue: Option<bool>,
377    pub run_closed_reason: Option<String>,
378    pub body: Value,
379}
380
381/// Stable validation categories for [`ActivityOptions`].
382#[derive(Clone, Copy, Debug, PartialEq, Eq)]
383pub enum ActivityOptionsErrorKind {
384    EmptyTaskQueue,
385    EmptyRetryPolicy,
386    InvalidMaxAttempts,
387    BackoffWithoutRetryBudget,
388    TooManyBackoffIntervals,
389    InvalidBackoffCoefficient,
390    BackoffGenerationTooLarge,
391    BackoffOverflow,
392    EmptyNonRetryableErrorType,
393    TimeoutNotPositive,
394    TimeoutOverflow,
395    TimeoutOrder,
396}
397
398/// A machine-readable activity-options validation failure.
399#[derive(Clone, Debug, Error, PartialEq, Eq)]
400#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
401pub struct ActivityOptionsError {
402    pub kind: ActivityOptionsErrorKind,
403    pub field: Option<&'static str>,
404    pub message: String,
405}
406
407impl ActivityOptionsError {
408    fn new(
409        kind: ActivityOptionsErrorKind,
410        field: Option<&'static str>,
411        message: impl Into<String>,
412    ) -> Self {
413        Self {
414            kind,
415            field,
416            message: message.into(),
417        }
418    }
419}
420
421/// Stable terminal categories returned when an awaited activity does not succeed.
422#[derive(Clone, Copy, Debug, PartialEq, Eq)]
423pub enum ActivityFailureKind {
424    Failed,
425    Cancelled,
426    TimedOut,
427}
428
429/// A stable, machine-readable terminal activity failure.
430///
431/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
432/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
433#[derive(Clone, Debug, Error)]
434#[error("activity failed ({reason}): {message}")]
435pub struct ActivityFailure {
436    pub kind: ActivityFailureKind,
437    pub reason: String,
438    pub message: String,
439    pub activity_execution_id: Option<String>,
440    pub activity_attempt_id: Option<String>,
441    pub activity_type: Option<String>,
442    pub activity_class: Option<String>,
443    pub attempt_number: Option<u64>,
444    pub failure_id: Option<String>,
445    pub failure_category: Option<String>,
446    pub timeout_kind: Option<String>,
447    pub non_retryable: bool,
448    pub exception_type: Option<String>,
449    pub exception_class: Option<String>,
450    pub code: Option<Value>,
451    pub exception: Option<Value>,
452}
453
454/// Stable terminal categories returned when an awaited child does not succeed.
455#[derive(Clone, Copy, Debug, PartialEq, Eq)]
456pub enum ChildWorkflowFailureKind {
457    Failed,
458    Cancelled,
459    Terminated,
460}
461
462/// A stable, machine-readable child workflow failure delivered to its parent.
463///
464/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
465/// of parsing the display message. Child and parent identifiers retain the
466/// relationship recorded in durable history across worker restarts.
467#[derive(Clone, Debug, Error)]
468#[error("child workflow failed ({reason}): {message}")]
469pub struct ChildWorkflowFailure {
470    pub kind: ChildWorkflowFailureKind,
471    pub reason: String,
472    pub message: String,
473    pub parent_workflow_id: Option<String>,
474    pub parent_workflow_run_id: Option<String>,
475    pub child_workflow_id: Option<String>,
476    pub child_workflow_run_id: Option<String>,
477    pub child_workflow_type: Option<String>,
478    pub failure_id: Option<String>,
479    pub failure_category: Option<String>,
480    pub exception_type: Option<String>,
481    pub exception_class: Option<String>,
482    pub non_retryable: bool,
483    pub code: Option<Value>,
484    pub exception: Option<Value>,
485}
486
487/// The identity of one durable workflow execution.
488#[derive(Clone, Debug, PartialEq, Eq)]
489pub struct WorkflowIdentity {
490    pub workflow_id: Option<String>,
491    pub run_id: Option<String>,
492}
493
494/// A successful child result together with its durable parent-child identity.
495#[derive(Clone, Debug, PartialEq)]
496pub struct ChildWorkflowResult {
497    pub parent: WorkflowIdentity,
498    pub child: WorkflowIdentity,
499    pub child_workflow_type: Option<String>,
500    pub result: Value,
501}
502
503/// Lossless successful child result for fixed Avro Value workflows.
504#[derive(Clone, Debug, PartialEq)]
505pub struct ChildWorkflowAvroResult {
506    pub parent: WorkflowIdentity,
507    pub child: WorkflowIdentity,
508    pub child_workflow_type: Option<String>,
509    pub result: AvroValue,
510}
511
512/// Server behavior when a parent closes while its child is still open.
513#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
514pub enum ParentClosePolicy {
515    #[default]
516    Abandon,
517    RequestCancel,
518    Terminate,
519}
520
521impl ParentClosePolicy {
522    fn as_str(self) -> &'static str {
523        match self {
524            Self::Abandon => "abandon",
525            Self::RequestCancel => "request_cancel",
526            Self::Terminate => "terminate",
527        }
528    }
529}
530
531/// Durable retry policy for one child workflow invocation.
532#[derive(Clone, Debug, Default, PartialEq, Eq)]
533pub struct ChildWorkflowRetryPolicy {
534    pub max_attempts: Option<u32>,
535    pub backoff_seconds: Vec<u64>,
536    pub non_retryable_error_types: Vec<String>,
537}
538
539/// Options recorded with a child-workflow command.
540///
541/// The task queue is mandatory so routing is explicit and replay-stable.
542#[derive(Clone, Debug, PartialEq, Eq)]
543pub struct ChildWorkflowOptions {
544    pub task_queue: String,
545    pub parent_close_policy: ParentClosePolicy,
546    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
547    pub execution_timeout_seconds: Option<u64>,
548    pub run_timeout_seconds: Option<u64>,
549}
550
551impl ChildWorkflowOptions {
552    pub fn new(task_queue: impl Into<String>) -> Self {
553        Self {
554            task_queue: task_queue.into(),
555            parent_close_policy: ParentClosePolicy::Abandon,
556            retry_policy: None,
557            execution_timeout_seconds: None,
558            run_timeout_seconds: None,
559        }
560    }
561
562    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
563        self.parent_close_policy = policy;
564        self
565    }
566
567    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
568        self.retry_policy = Some(policy);
569        self
570    }
571
572    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
573        self.execution_timeout_seconds = Some(seconds);
574        self
575    }
576
577    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
578        self.run_timeout_seconds = Some(seconds);
579        self
580    }
581}
582
583/// Backoff intervals for one durable activity retry policy.
584#[derive(Clone, Debug, PartialEq, Eq)]
585pub enum ActivityBackoff {
586    /// Use these intervals between attempts. The server repeats the final
587    /// interval if the retry budget contains more attempts than entries.
588    Explicit(Vec<Duration>),
589    /// Generate one interval for every retry using integer exponential growth.
590    Exponential {
591        initial_interval: Duration,
592        coefficient: u32,
593        maximum_interval: Option<Duration>,
594    },
595}
596
597/// Durable server-side retry policy for one activity execution.
598#[derive(Clone, Debug, Default, PartialEq, Eq)]
599pub struct ActivityRetryPolicy {
600    pub max_attempts: Option<u32>,
601    pub backoff: Option<ActivityBackoff>,
602    pub non_retryable_error_types: Vec<String>,
603}
604
605impl ActivityRetryPolicy {
606    /// Start a policy with a finite attempt budget, including the first attempt.
607    pub fn new(max_attempts: u32) -> Self {
608        Self {
609            max_attempts: Some(max_attempts),
610            ..Self::default()
611        }
612    }
613
614    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
615        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
616        self
617    }
618
619    pub fn exponential_backoff(
620        mut self,
621        initial_interval: Duration,
622        coefficient: u32,
623        maximum_interval: Option<Duration>,
624    ) -> Self {
625        self.backoff = Some(ActivityBackoff::Exponential {
626            initial_interval,
627            coefficient,
628            maximum_interval,
629        });
630        self
631    }
632
633    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
634        self.non_retryable_error_types.push(error_type.into());
635        self
636    }
637
638    pub fn non_retryable_error_types(
639        mut self,
640        error_types: impl IntoIterator<Item = impl Into<String>>,
641    ) -> Self {
642        self.non_retryable_error_types
643            .extend(error_types.into_iter().map(Into::into));
644        self
645    }
646}
647
648/// Options recorded atomically on one deterministic `schedule_activity` command.
649///
650/// Durations are rounded up to whole seconds when encoded, so the server never
651/// receives a shorter timeout or backoff than the caller requested.
652#[derive(Clone, Debug, Default, PartialEq, Eq)]
653pub struct ActivityOptions {
654    pub task_queue: Option<String>,
655    pub retry_policy: Option<ActivityRetryPolicy>,
656    pub start_to_close_timeout: Option<Duration>,
657    pub schedule_to_start_timeout: Option<Duration>,
658    pub schedule_to_close_timeout: Option<Duration>,
659    pub heartbeat_timeout: Option<Duration>,
660}
661
662impl ActivityOptions {
663    pub fn new() -> Self {
664        Self::default()
665    }
666
667    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
668        self.task_queue = Some(task_queue.into());
669        self
670    }
671
672    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
673        self.retry_policy = Some(policy);
674        self
675    }
676
677    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
678        self.start_to_close_timeout = Some(timeout);
679        self
680    }
681
682    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
683        self.schedule_to_start_timeout = Some(timeout);
684        self
685    }
686
687    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
688        self.schedule_to_close_timeout = Some(timeout);
689        self
690    }
691
692    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
693        self.heartbeat_timeout = Some(timeout);
694        self
695    }
696
697    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
698        if self
699            .task_queue
700            .as_deref()
701            .is_some_and(|queue| queue.trim().is_empty())
702        {
703            return Err(ActivityOptionsError::new(
704                ActivityOptionsErrorKind::EmptyTaskQueue,
705                Some("task_queue"),
706                "task_queue must not be empty",
707            ));
708        }
709
710        for (field, value) in [
711            ("start_to_close_timeout", self.start_to_close_timeout),
712            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
713            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
714            ("heartbeat_timeout", self.heartbeat_timeout),
715        ] {
716            if value.is_some_and(|value| value.is_zero()) {
717                return Err(ActivityOptionsError::new(
718                    ActivityOptionsErrorKind::TimeoutNotPositive,
719                    Some(field),
720                    format!("{field} must be positive"),
721                ));
722            }
723        }
724
725        validate_timeout_order(
726            "heartbeat_timeout",
727            self.heartbeat_timeout,
728            "start_to_close_timeout",
729            self.start_to_close_timeout,
730        )?;
731        validate_timeout_order(
732            "start_to_close_timeout",
733            self.start_to_close_timeout,
734            "schedule_to_close_timeout",
735            self.schedule_to_close_timeout,
736        )?;
737        validate_timeout_order(
738            "schedule_to_start_timeout",
739            self.schedule_to_start_timeout,
740            "schedule_to_close_timeout",
741            self.schedule_to_close_timeout,
742        )?;
743
744        Ok(ValidatedActivityOptions {
745            task_queue: self.task_queue.clone(),
746            retry_policy: self
747                .retry_policy
748                .as_ref()
749                .map(validate_activity_retry_policy)
750                .transpose()?,
751            start_to_close_timeout: timeout_seconds(
752                "start_to_close_timeout",
753                self.start_to_close_timeout,
754            )?,
755            schedule_to_start_timeout: timeout_seconds(
756                "schedule_to_start_timeout",
757                self.schedule_to_start_timeout,
758            )?,
759            schedule_to_close_timeout: timeout_seconds(
760                "schedule_to_close_timeout",
761                self.schedule_to_close_timeout,
762            )?,
763            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
764        })
765    }
766}
767
768#[derive(Clone, Debug)]
769struct ValidatedActivityOptions {
770    task_queue: Option<String>,
771    retry_policy: Option<Value>,
772    start_to_close_timeout: Option<u64>,
773    schedule_to_start_timeout: Option<u64>,
774    schedule_to_close_timeout: Option<u64>,
775    heartbeat_timeout: Option<u64>,
776}
777
778fn validate_timeout_order(
779    smaller_name: &'static str,
780    smaller: Option<Duration>,
781    larger_name: &'static str,
782    larger: Option<Duration>,
783) -> std::result::Result<(), ActivityOptionsError> {
784    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
785        return Err(ActivityOptionsError::new(
786            ActivityOptionsErrorKind::TimeoutOrder,
787            Some(smaller_name),
788            format!("{smaller_name} must be <= {larger_name}"),
789        ));
790    }
791    Ok(())
792}
793
794fn timeout_seconds(
795    field: &'static str,
796    value: Option<Duration>,
797) -> std::result::Result<Option<u64>, ActivityOptionsError> {
798    value
799        .map(|value| {
800            activity_protocol_seconds(value).ok_or_else(|| {
801                ActivityOptionsError::new(
802                    ActivityOptionsErrorKind::TimeoutOverflow,
803                    Some(field),
804                    format!("{field} is too large for the worker protocol"),
805                )
806            })
807        })
808        .transpose()
809}
810
811fn duration_seconds_ceil(value: Duration) -> Option<u64> {
812    value
813        .as_secs()
814        .checked_add(u64::from(value.subsec_nanos() > 0))
815}
816
817fn activity_protocol_seconds(value: Duration) -> Option<u64> {
818    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
819}
820
821fn validate_activity_retry_policy(
822    policy: &ActivityRetryPolicy,
823) -> std::result::Result<Value, ActivityOptionsError> {
824    if policy.max_attempts.is_none()
825        && policy.backoff.is_none()
826        && policy.non_retryable_error_types.is_empty()
827    {
828        return Err(ActivityOptionsError::new(
829            ActivityOptionsErrorKind::EmptyRetryPolicy,
830            Some("retry_policy"),
831            "retry_policy must configure at least one field",
832        ));
833    }
834    if policy.max_attempts == Some(0) {
835        return Err(ActivityOptionsError::new(
836            ActivityOptionsErrorKind::InvalidMaxAttempts,
837            Some("retry_policy.max_attempts"),
838            "max_attempts must be >= 1",
839        ));
840    }
841    if policy
842        .non_retryable_error_types
843        .iter()
844        .any(|error_type| error_type.trim().is_empty())
845    {
846        return Err(ActivityOptionsError::new(
847            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
848            Some("retry_policy.non_retryable_error_types"),
849            "non_retryable_error_types must not contain empty values",
850        ));
851    }
852
853    let backoff_seconds = match &policy.backoff {
854        None => None,
855        Some(backoff) => {
856            let max_attempts = policy.max_attempts.ok_or_else(|| {
857                ActivityOptionsError::new(
858                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
859                    Some("retry_policy.backoff"),
860                    "backoff requires max_attempts",
861                )
862            })?;
863            let retry_count = max_attempts.saturating_sub(1) as usize;
864            let intervals = match backoff {
865                ActivityBackoff::Explicit(intervals) => {
866                    if intervals.len() > retry_count {
867                        return Err(ActivityOptionsError::new(
868                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
869                            Some("retry_policy.backoff"),
870                            "backoff interval count must not exceed max_attempts - 1",
871                        ));
872                    }
873                    intervals.clone()
874                }
875                ActivityBackoff::Exponential {
876                    initial_interval,
877                    coefficient,
878                    maximum_interval,
879                } => {
880                    if *coefficient < 1 {
881                        return Err(ActivityOptionsError::new(
882                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
883                            Some("retry_policy.backoff.coefficient"),
884                            "backoff coefficient must be >= 1",
885                        ));
886                    }
887                    if retry_count > 10_000 {
888                        return Err(ActivityOptionsError::new(
889                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
890                            Some("retry_policy.max_attempts"),
891                            "generated backoff supports at most 10000 retry intervals",
892                        ));
893                    }
894                    let mut current = *initial_interval;
895                    let mut intervals = Vec::with_capacity(retry_count);
896                    for _ in 0..retry_count {
897                        let interval = maximum_interval
898                            .map(|maximum| current.min(maximum))
899                            .unwrap_or(current);
900                        intervals.push(interval);
901                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
902                            break;
903                        }
904                        current = current.checked_mul(*coefficient).ok_or_else(|| {
905                            ActivityOptionsError::new(
906                                ActivityOptionsErrorKind::BackoffOverflow,
907                                Some("retry_policy.backoff"),
908                                "generated backoff interval overflowed",
909                            )
910                        })?;
911                    }
912                    intervals
913                }
914            };
915            Some(
916                intervals
917                    .into_iter()
918                    .map(|interval| {
919                        activity_protocol_seconds(interval).ok_or_else(|| {
920                            ActivityOptionsError::new(
921                                ActivityOptionsErrorKind::BackoffOverflow,
922                                Some("retry_policy.backoff"),
923                                "backoff interval is too large for the worker protocol",
924                            )
925                        })
926                    })
927                    .collect::<std::result::Result<Vec<_>, _>>()?,
928            )
929        }
930    };
931
932    let mut encoded = serde_json::Map::new();
933    if let Some(max_attempts) = policy.max_attempts {
934        encoded.insert("max_attempts".to_string(), json!(max_attempts));
935    }
936    if let Some(backoff_seconds) = backoff_seconds {
937        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
938    }
939    if !policy.non_retryable_error_types.is_empty() {
940        let mut canonical_error_types = Vec::new();
941        for error_type in policy
942            .non_retryable_error_types
943            .iter()
944            .map(|error_type| error_type.trim())
945        {
946            if !canonical_error_types.contains(&error_type) {
947                canonical_error_types.push(error_type);
948            }
949        }
950        encoded.insert(
951            "non_retryable_error_types".to_string(),
952            json!(canonical_error_types),
953        );
954    }
955    Ok(Value::Object(encoded))
956}
957
958/// A stable, machine-readable failure raised when workflow code no longer
959/// reconstructs the durable command stream recorded in history.
960#[derive(Clone, Debug, Error)]
961#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
962pub struct ReplayFailure {
963    pub reason: String,
964    pub sequence: Option<u64>,
965    pub expected: Option<String>,
966    pub actual: Option<String>,
967    pub message: String,
968}
969
970impl ReplayFailure {
971    fn new(
972        reason: impl Into<String>,
973        sequence: Option<u64>,
974        expected: Option<String>,
975        actual: Option<String>,
976        message: impl Into<String>,
977    ) -> Self {
978        Self {
979            reason: reason.into(),
980            sequence,
981            expected,
982            actual,
983            message: message.into(),
984        }
985    }
986}
987
988/// A stable, machine-readable workflow query or query-task settlement failure.
989#[derive(Clone, Debug, Error)]
990#[error("query failed ({reason}, HTTP {status}): {message}")]
991pub struct QueryFailure {
992    pub status: u16,
993    pub reason: String,
994    pub message: String,
995    pub body: Value,
996}
997
998/// A stable failure returned when a server rejects an SDK protocol version.
999#[derive(Clone, Debug, Error)]
1000#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1001pub struct ProtocolFailure {
1002    pub status: u16,
1003    pub reason: String,
1004    pub message: String,
1005    pub supported_version: Option<String>,
1006    pub requested_version: Option<String>,
1007    pub body: Value,
1008}
1009
1010#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1011pub struct PayloadEnvelope {
1012    pub codec: String,
1013    pub blob: String,
1014}
1015
1016impl PayloadEnvelope {
1017    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1018        encode_payload(value, DEFAULT_CODEC)
1019    }
1020
1021    pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1022        encode_payload(value, JSON_CODEC)
1023    }
1024
1025    /// Encode an explicit typed value, including the bytes branch that JSON
1026    /// serialization cannot represent.
1027    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1028        encode_avro_value(value)
1029    }
1030}
1031
1032/// Native adapter for the fixed language-neutral Avro Value schema.
1033#[derive(Clone, Debug, PartialEq)]
1034pub enum AvroValue {
1035    Null,
1036    Boolean(bool),
1037    Long(i64),
1038    Double(f64),
1039    Bytes(Vec<u8>),
1040    String(String),
1041    Array(Vec<AvroValue>),
1042    Map(BTreeMap<String, AvroValue>),
1043}
1044
1045impl AvroValue {
1046    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1047        Self::from_serde_value(
1048            serde_value::to_value(value).map_err(|error| {
1049                Error::Codec(format!("could not adapt value for Avro: {error}"))
1050            })?,
1051        )
1052    }
1053
1054    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1055        use serde_value::Value as SerdeValue;
1056
1057        match value {
1058            SerdeValue::Unit => Ok(Self::Null),
1059            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1060            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1061            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1062            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1063            SerdeValue::I64(value) => Ok(Self::Long(value)),
1064            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1065            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1066            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1067            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1068                Error::Codec(
1069                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1070                        .to_string(),
1071                )
1072            }),
1073            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1074            SerdeValue::F64(value) => Self::finite_double(value),
1075            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1076            SerdeValue::String(value) => Ok(Self::String(value)),
1077            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1078            SerdeValue::Option(None) => Ok(Self::Null),
1079            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1080                Self::from_serde_value(*value)
1081            }
1082            SerdeValue::Seq(values) => values
1083                .into_iter()
1084                .map(Self::from_serde_value)
1085                .collect::<Result<Vec<_>>>()
1086                .map(Self::Array),
1087            SerdeValue::Map(values) => values
1088                .into_iter()
1089                .map(|(key, value)| {
1090                    let SerdeValue::String(key) = key else {
1091                        return Err(Error::Codec(
1092                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1093                        ));
1094                    };
1095
1096                    Ok((key, Self::from_serde_value(value)?))
1097                })
1098                .collect::<Result<BTreeMap<_, _>>>()
1099                .map(Self::Map),
1100        }
1101    }
1102
1103    fn finite_double(value: f64) -> Result<Self> {
1104        if !value.is_finite() {
1105            return Err(Error::Codec(
1106                "non_finite_float: Avro Value doubles must be finite".to_string(),
1107            ));
1108        }
1109
1110        Ok(Self::Double(value))
1111    }
1112
1113    fn into_json(self) -> Result<Value> {
1114        match self {
1115            Self::Null => Ok(Value::Null),
1116            Self::Boolean(value) => Ok(Value::Bool(value)),
1117            Self::Long(value) => Ok(Value::Number(value.into())),
1118            Self::Double(value) => serde_json::Number::from_f64(value)
1119                .map(Value::Number)
1120                .ok_or_else(|| {
1121                    Error::Codec(
1122                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1123                    )
1124                }),
1125            Self::Bytes(value) => Ok(json!({
1126                "$type": "bytes",
1127                "base64": BASE64.encode(value),
1128            })),
1129            Self::String(value) => Ok(Value::String(value)),
1130            Self::Array(values) => values
1131                .into_iter()
1132                .map(Self::into_json)
1133                .collect::<Result<Vec<_>>>()
1134                .map(Value::Array),
1135            Self::Map(values) => values
1136                .into_iter()
1137                .map(|(key, value)| Ok((key, value.into_json()?)))
1138                .collect::<Result<serde_json::Map<_, _>>>()
1139                .map(Value::Object),
1140        }
1141    }
1142
1143    fn into_serde_value(self) -> serde_value::Value {
1144        use serde_value::Value as SerdeValue;
1145
1146        match self {
1147            Self::Null => SerdeValue::Unit,
1148            Self::Boolean(value) => SerdeValue::Bool(value),
1149            Self::Long(value) => SerdeValue::I64(value),
1150            Self::Double(value) => SerdeValue::F64(value),
1151            Self::Bytes(value) => SerdeValue::Bytes(value),
1152            Self::String(value) => SerdeValue::String(value),
1153            Self::Array(values) => {
1154                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1155            }
1156            Self::Map(values) => SerdeValue::Map(
1157                values
1158                    .into_iter()
1159                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1160                    .collect(),
1161            ),
1162        }
1163    }
1164
1165    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1166        self.into_serde_value().deserialize_into().map_err(|error| {
1167            Error::Codec(format!(
1168                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1169            ))
1170        })
1171    }
1172}
1173
1174impl Serialize for AvroValue {
1175    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1176    where
1177        S: Serializer,
1178    {
1179        match self {
1180            Self::Null => serializer.serialize_unit(),
1181            Self::Boolean(value) => serializer.serialize_bool(*value),
1182            Self::Long(value) => serializer.serialize_i64(*value),
1183            Self::Double(value) => serializer.serialize_f64(*value),
1184            Self::Bytes(value) => serializer.serialize_bytes(value),
1185            Self::String(value) => serializer.serialize_str(value),
1186            Self::Array(values) => {
1187                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1188                for value in values {
1189                    sequence.serialize_element(value)?;
1190                }
1191                sequence.end()
1192            }
1193            Self::Map(values) => {
1194                let mut map = serializer.serialize_map(Some(values.len()))?;
1195                for (key, value) in values {
1196                    map.serialize_entry(key, value)?;
1197                }
1198                map.end()
1199            }
1200        }
1201    }
1202}
1203
1204pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1205    let datum = avro_value_to_datum(value)?;
1206    let datum = to_avro_datum(avro_value_schema()?, datum)
1207        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1208    let mut bytes = Vec::with_capacity(datum.len() + 10);
1209    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1210    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1211    bytes.extend_from_slice(&datum);
1212    Ok(PayloadEnvelope {
1213        codec: DEFAULT_CODEC.to_string(),
1214        blob: BASE64.encode(bytes),
1215    })
1216}
1217
1218pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1219    if envelope.codec != DEFAULT_CODEC {
1220        return Err(Error::Codec(format!(
1221            "unsupported payload codec {:?}",
1222            envelope.codec
1223        )));
1224    }
1225    decode_avro_value_blob(&envelope.blob)
1226}
1227
1228pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1229    let blob = match codec {
1230        JSON_CODEC => serde_json::to_string(value)?,
1231        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1232        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1233    };
1234
1235    Ok(PayloadEnvelope {
1236        codec: codec.to_string(),
1237        blob,
1238    })
1239}
1240
1241pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1242    match envelope.codec.as_str() {
1243        JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1244        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1245        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1246    }
1247}
1248
1249#[cfg(test)]
1250fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1251    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1252}
1253
1254fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1255    if value.is_null() {
1256        return Ok(Value::Null);
1257    }
1258
1259    if let Some(object) = value.as_object() {
1260        if let (Some(codec), Some(blob)) = (
1261            object.get("codec").and_then(Value::as_str),
1262            object.get("blob").and_then(Value::as_str),
1263        ) {
1264            return decode_blob(blob, codec);
1265        }
1266    }
1267
1268    if let Some(blob) = value.as_str() {
1269        return decode_blob(blob, fallback_codec);
1270    }
1271
1272    Ok(value.clone())
1273}
1274
1275fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1276    let envelope = match codec {
1277        DEFAULT_CODEC => encode_avro_value(value)?,
1278        JSON_CODEC => PayloadEnvelope {
1279            codec: JSON_CODEC.to_string(),
1280            blob: serde_json::to_string(&value.clone().into_json()?)?,
1281        },
1282        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1283    };
1284    Ok(serde_json::to_value(envelope)?)
1285}
1286
1287fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1288    if value.is_null() {
1289        return Ok(AvroValue::Null);
1290    }
1291
1292    if let Some(object) = value.as_object() {
1293        if let (Some(codec), Some(blob)) = (
1294            object.get("codec").and_then(Value::as_str),
1295            object.get("blob").and_then(Value::as_str),
1296        ) {
1297            return match codec {
1298                DEFAULT_CODEC => decode_avro_value_blob(blob),
1299                JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1300                other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1301            };
1302        }
1303    }
1304
1305    if let Some(blob) = value.as_str() {
1306        return match fallback_codec {
1307            DEFAULT_CODEC => decode_avro_value_blob(blob),
1308            JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1309            other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1310        };
1311    }
1312
1313    AvroValue::from_serialize(value)
1314}
1315
1316fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1317    match value {
1318        AvroValue::Null => AvroValue::Array(Vec::new()),
1319        AvroValue::Array(_) => value,
1320        other => AvroValue::Array(vec![other]),
1321    }
1322}
1323
1324fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1325    match codec {
1326        JSON_CODEC => Ok(serde_json::from_str(blob)?),
1327        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1328        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1329    }
1330}
1331
1332fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1333    let bytes = BASE64.decode(blob).map_err(|err| {
1334        Error::Codec(format!(
1335            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1336        ))
1337    })?;
1338
1339    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1340        return Err(Error::Codec(
1341            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1342        ));
1343    }
1344
1345    let fingerprint: [u8; 8] = bytes[2..10]
1346        .try_into()
1347        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1348    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1349        return Err(Error::Codec(format!(
1350            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1351            fingerprint
1352                .iter()
1353                .map(|byte| format!("{byte:02x}"))
1354                .collect::<String>()
1355        )));
1356    }
1357
1358    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1359    // The current fingerprint selects the current immutable schema, so reader
1360    // resolution would only re-walk the same recursive union. Future retained
1361    // writer fingerprints supply a distinct reader schema in this branch.
1362    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1363    if datum_reader.truncated {
1364        return Err(Error::Codec(
1365            "invalid_payload_framing: truncated Avro Value datum".to_string(),
1366        ));
1367    }
1368    let datum = datum.map_err(|err| {
1369        Error::Codec(format!(
1370            "invalid_payload_framing: malformed Avro Value datum: {err}"
1371        ))
1372    })?;
1373    if datum_reader.remaining() != 0 {
1374        return Err(Error::Codec(format!(
1375            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1376            datum_reader.remaining()
1377        )));
1378    }
1379    avro_value_from_datum(datum)
1380}
1381
1382struct StrictAvroDatumReader<'a> {
1383    bytes: &'a [u8],
1384    offset: usize,
1385    truncated: bool,
1386}
1387
1388impl<'a> StrictAvroDatumReader<'a> {
1389    fn new(bytes: &'a [u8]) -> Self {
1390        Self {
1391            bytes,
1392            offset: 0,
1393            truncated: false,
1394        }
1395    }
1396
1397    fn remaining(&self) -> usize {
1398        self.bytes.len() - self.offset
1399    }
1400}
1401
1402impl Read for StrictAvroDatumReader<'_> {
1403    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1404        let count = buffer.len().min(self.remaining());
1405        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1406        self.offset += count;
1407        if count < buffer.len() {
1408            self.truncated = true;
1409        }
1410
1411        Ok(count)
1412    }
1413}
1414
1415fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1416    let branch = match value {
1417        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1418        AvroValue::Boolean(value) => AvroDatum::Union(
1419            1,
1420            Box::new(AvroDatum::Record(vec![(
1421                "boolean".to_string(),
1422                AvroDatum::Boolean(*value),
1423            )])),
1424        ),
1425        AvroValue::Long(value) => AvroDatum::Union(
1426            2,
1427            Box::new(AvroDatum::Record(vec![(
1428                "long".to_string(),
1429                AvroDatum::Long(*value),
1430            )])),
1431        ),
1432        AvroValue::Double(value) => {
1433            if !value.is_finite() {
1434                return Err(Error::Codec(
1435                    "non_finite_float: Avro Value doubles must be finite".to_string(),
1436                ));
1437            }
1438            AvroDatum::Union(
1439                3,
1440                Box::new(AvroDatum::Record(vec![(
1441                    "double".to_string(),
1442                    AvroDatum::Double(*value),
1443                )])),
1444            )
1445        }
1446        AvroValue::Bytes(value) => AvroDatum::Union(
1447            4,
1448            Box::new(AvroDatum::Record(vec![(
1449                "bytes".to_string(),
1450                AvroDatum::Bytes(value.clone()),
1451            )])),
1452        ),
1453        AvroValue::String(value) => AvroDatum::Union(
1454            5,
1455            Box::new(AvroDatum::Record(vec![(
1456                "string".to_string(),
1457                AvroDatum::String(value.clone()),
1458            )])),
1459        ),
1460        AvroValue::Array(values) => AvroDatum::Union(
1461            6,
1462            Box::new(AvroDatum::Record(vec![(
1463                "items".to_string(),
1464                AvroDatum::Array(
1465                    values
1466                        .iter()
1467                        .map(avro_value_to_datum)
1468                        .collect::<Result<Vec<_>>>()?,
1469                ),
1470            )])),
1471        ),
1472        AvroValue::Map(values) => AvroDatum::Union(
1473            7,
1474            Box::new(AvroDatum::Record(vec![(
1475                "entries".to_string(),
1476                AvroDatum::Map(
1477                    values
1478                        .iter()
1479                        .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1480                        .collect::<Result<HashMap<_, _>>>()?,
1481                ),
1482            )])),
1483        ),
1484    };
1485    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1486}
1487
1488fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1489    let AvroDatum::Record(mut outer) = datum else {
1490        return Err(Error::Codec(
1491            "invalid_payload_framing: datum is not a Value record".to_string(),
1492        ));
1493    };
1494    let (_, branch) = outer
1495        .pop()
1496        .filter(|(name, _)| name == "value")
1497        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1498    let AvroDatum::Union(_, branch) = branch else {
1499        return Err(Error::Codec(
1500            "invalid_payload_framing: invalid Value union".to_string(),
1501        ));
1502    };
1503    match *branch {
1504        AvroDatum::Null => Ok(AvroValue::Null),
1505        AvroDatum::Record(mut fields) => {
1506            let (name, value) = fields.pop().ok_or_else(|| {
1507                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1508            })?;
1509            match (name.as_str(), value) {
1510                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1511                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1512                ("double", AvroDatum::Double(value)) if value.is_finite() => {
1513                    Ok(AvroValue::Double(value))
1514                }
1515                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1516                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1517                ("items", AvroDatum::Array(values)) => values
1518                    .into_iter()
1519                    .map(avro_value_from_datum)
1520                    .collect::<Result<Vec<_>>>()
1521                    .map(AvroValue::Array),
1522                ("entries", AvroDatum::Map(values)) => values
1523                    .into_iter()
1524                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1525                    .collect::<Result<BTreeMap<_, _>>>()
1526                    .map(AvroValue::Map),
1527                _ => Err(Error::Codec(
1528                    "invalid_payload_framing: unknown Value branch".to_string(),
1529                )),
1530            }
1531        }
1532        _ => Err(Error::Codec(
1533            "invalid_payload_framing: invalid Value branch".to_string(),
1534        )),
1535    }
1536}
1537
1538fn avro_value_schema() -> Result<&'static Schema> {
1539    match AVRO_VALUE_SCHEMA.get_or_init(|| {
1540        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1541            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1542    }) {
1543        Ok(schema) => Ok(schema),
1544        Err(message) => Err(Error::Codec(message.clone())),
1545    }
1546}
1547
1548#[derive(Clone, Debug)]
1549pub struct Client {
1550    http: reqwest::Client,
1551    base_url: String,
1552    token: Option<String>,
1553    control_token: Option<String>,
1554    worker_token: Option<String>,
1555    namespace: String,
1556}
1557
1558impl Client {
1559    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1560        Self::builder(base_url).build()
1561    }
1562
1563    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1564        ClientBuilder {
1565            base_url: base_url.into(),
1566            token: None,
1567            control_token: None,
1568            worker_token: None,
1569            namespace: "default".to_string(),
1570            timeout: Duration::from_secs(60),
1571        }
1572    }
1573
1574    pub async fn health(&self) -> Result<Value> {
1575        self.request_json(
1576            reqwest::Method::GET,
1577            "/health",
1578            RequestProtocol::ControlPlane,
1579            Option::<&Value>::None,
1580        )
1581        .await
1582    }
1583
1584    pub async fn cluster_info(&self) -> Result<Value> {
1585        self.request_json(
1586            reqwest::Method::GET,
1587            "/cluster/info",
1588            RequestProtocol::ControlPlane,
1589            Option::<&Value>::None,
1590        )
1591        .await
1592    }
1593
1594    pub async fn start_workflow<T: Serialize>(
1595        &self,
1596        workflow_type: &str,
1597        task_queue: &str,
1598        workflow_id: &str,
1599        input: T,
1600    ) -> Result<WorkflowHandle> {
1601        self.start_workflow_with_options(
1602            workflow_type,
1603            task_queue,
1604            workflow_id,
1605            WorkflowStartOptions::default(),
1606            input,
1607        )
1608        .await
1609    }
1610
1611    /// Start a workflow with explicit server-enforced execution and run
1612    /// deadlines.
1613    pub async fn start_workflow_with_options<T: Serialize>(
1614        &self,
1615        workflow_type: &str,
1616        task_queue: &str,
1617        workflow_id: &str,
1618        options: WorkflowStartOptions,
1619        input: T,
1620    ) -> Result<WorkflowHandle> {
1621        options.validate()?;
1622        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1623        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1624        let body = json!({
1625            "workflow_id": workflow_id,
1626            "workflow_type": workflow_type,
1627            "task_queue": task_queue,
1628            "input": input_envelope,
1629            "execution_timeout_seconds": options.execution_timeout_seconds,
1630            "run_timeout_seconds": options.run_timeout_seconds
1631        });
1632
1633        let data: Value = self
1634            .request_json(
1635                reqwest::Method::POST,
1636                "/workflows",
1637                RequestProtocol::ControlPlane,
1638                Some(&body),
1639            )
1640            .await?;
1641
1642        Ok(WorkflowHandle {
1643            client: self.clone(),
1644            workflow_id: data
1645                .get("workflow_id")
1646                .and_then(Value::as_str)
1647                .unwrap_or(workflow_id)
1648                .to_string(),
1649            run_id: data
1650                .get("run_id")
1651                .and_then(Value::as_str)
1652                .map(str::to_string),
1653            workflow_type: data
1654                .get("workflow_type")
1655                .and_then(Value::as_str)
1656                .unwrap_or(workflow_type)
1657                .to_string(),
1658        })
1659    }
1660
1661    pub async fn signal_workflow<T: Serialize>(
1662        &self,
1663        workflow_id: &str,
1664        signal_name: &str,
1665        input: T,
1666    ) -> Result<Value> {
1667        self.signal_workflow_target(workflow_id, None, signal_name, input)
1668            .await
1669    }
1670
1671    /// Signal only if `run_id` is still the current run for this instance.
1672    pub async fn signal_workflow_run<T: Serialize>(
1673        &self,
1674        workflow_id: &str,
1675        run_id: &str,
1676        signal_name: &str,
1677        input: T,
1678    ) -> Result<Value> {
1679        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1680            .await
1681    }
1682
1683    async fn signal_workflow_target<T: Serialize>(
1684        &self,
1685        workflow_id: &str,
1686        run_id: Option<&str>,
1687        signal_name: &str,
1688        input: T,
1689    ) -> Result<Value> {
1690        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1691        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1692        let body = json!({
1693            "input": input_envelope
1694        });
1695        let path = match run_id {
1696            Some(run_id) => {
1697                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1698            }
1699            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1700        };
1701        self.request_json(
1702            reqwest::Method::POST,
1703            &path,
1704            RequestProtocol::ControlPlane,
1705            Some(&body),
1706        )
1707        .await
1708    }
1709
1710    /// Request cooperative cancellation of the current run for an instance.
1711    pub async fn cancel_workflow(
1712        &self,
1713        workflow_id: &str,
1714        options: WorkflowCommandOptions,
1715    ) -> Result<WorkflowCommandResult> {
1716        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1717            .await
1718    }
1719
1720    /// Request cooperative cancellation only if `run_id` is still current.
1721    pub async fn cancel_workflow_run(
1722        &self,
1723        workflow_id: &str,
1724        run_id: &str,
1725        options: WorkflowCommandOptions,
1726    ) -> Result<WorkflowCommandResult> {
1727        self.workflow_command(
1728            workflow_id,
1729            Some(run_id),
1730            WorkflowCommandKind::Cancel,
1731            options,
1732        )
1733        .await
1734    }
1735
1736    /// Forcefully terminate the current run for an instance.
1737    pub async fn terminate_workflow(
1738        &self,
1739        workflow_id: &str,
1740        options: WorkflowCommandOptions,
1741    ) -> Result<WorkflowCommandResult> {
1742        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1743            .await
1744    }
1745
1746    /// Forcefully terminate only if `run_id` is still current.
1747    pub async fn terminate_workflow_run(
1748        &self,
1749        workflow_id: &str,
1750        run_id: &str,
1751        options: WorkflowCommandOptions,
1752    ) -> Result<WorkflowCommandResult> {
1753        self.workflow_command(
1754            workflow_id,
1755            Some(run_id),
1756            WorkflowCommandKind::Terminate,
1757            options,
1758        )
1759        .await
1760    }
1761
1762    async fn workflow_command(
1763        &self,
1764        workflow_id: &str,
1765        run_id: Option<&str>,
1766        command: WorkflowCommandKind,
1767        options: WorkflowCommandOptions,
1768    ) -> Result<WorkflowCommandResult> {
1769        let path = match run_id {
1770            Some(run_id) => format!(
1771                "/workflows/{workflow_id}/runs/{run_id}/{}",
1772                command.as_str()
1773            ),
1774            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1775        };
1776        let data = match self
1777            .request_json(
1778                reqwest::Method::POST,
1779                &path,
1780                RequestProtocol::ControlPlane,
1781                Some(&options),
1782            )
1783            .await
1784        {
1785            Ok(data) => data,
1786            Err(Error::Http { status, body }) => {
1787                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1788                    command,
1789                    status,
1790                    body,
1791                    workflow_id,
1792                    run_id,
1793                )));
1794            }
1795            Err(error) => return Err(error),
1796        };
1797
1798        Ok(workflow_command_result(command, data, workflow_id, run_id))
1799    }
1800
1801    /// Execute a named, read-only query against a running or completed workflow.
1802    ///
1803    /// Arguments and results use the platform payload envelope. Server and
1804    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1805    /// reason, HTTP status, and original response body.
1806    pub async fn query_workflow<T: Serialize>(
1807        &self,
1808        workflow_id: &str,
1809        query_name: &str,
1810        input: T,
1811    ) -> Result<Value> {
1812        self.query_workflow_target(workflow_id, None, query_name, input)
1813            .await
1814    }
1815
1816    /// Query only if `run_id` is still current, preventing accidental retargeting.
1817    pub async fn query_workflow_run<T: Serialize>(
1818        &self,
1819        workflow_id: &str,
1820        run_id: &str,
1821        query_name: &str,
1822        input: T,
1823    ) -> Result<Value> {
1824        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1825            .await
1826    }
1827
1828    /// Query a workflow and return the lossless fixed Avro Value result.
1829    pub async fn query_workflow_avro_value<T: Serialize>(
1830        &self,
1831        workflow_id: &str,
1832        query_name: &str,
1833        input: T,
1834    ) -> Result<AvroValue> {
1835        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1836            .await
1837    }
1838
1839    /// Query a selected run and return the lossless fixed Avro Value result.
1840    pub async fn query_workflow_run_avro_value<T: Serialize>(
1841        &self,
1842        workflow_id: &str,
1843        run_id: &str,
1844        query_name: &str,
1845        input: T,
1846    ) -> Result<AvroValue> {
1847        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1848            .await
1849    }
1850
1851    async fn query_workflow_avro_value_target<T: Serialize>(
1852        &self,
1853        workflow_id: &str,
1854        run_id: Option<&str>,
1855        query_name: &str,
1856        input: T,
1857    ) -> Result<AvroValue> {
1858        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1859        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1860        let path = match run_id {
1861            Some(run_id) => {
1862                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1863            }
1864            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1865        };
1866        let response: Value = match self
1867            .request_json(
1868                reqwest::Method::POST,
1869                &path,
1870                RequestProtocol::ControlPlane,
1871                Some(&body),
1872            )
1873            .await
1874        {
1875            Ok(response) => response,
1876            Err(Error::Http { status, body }) => {
1877                return Err(Error::QueryFailed(query_failure(status, body)));
1878            }
1879            Err(error) => return Err(error),
1880        };
1881
1882        let envelope = response
1883            .get("result_envelope")
1884            .filter(|envelope| !envelope.is_null())
1885            .ok_or_else(|| {
1886                Error::Codec(
1887                    "missing_payload_envelope: typed query result requires result_envelope"
1888                        .to_string(),
1889                )
1890            })?;
1891        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1892    }
1893
1894    async fn query_workflow_target<T: Serialize>(
1895        &self,
1896        workflow_id: &str,
1897        run_id: Option<&str>,
1898        query_name: &str,
1899        input: T,
1900    ) -> Result<Value> {
1901        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1902        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1903        let body = json!({
1904            "input": input_envelope
1905        });
1906        let path = match run_id {
1907            Some(run_id) => {
1908                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1909            }
1910            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1911        };
1912        let response: Value = match self
1913            .request_json(
1914                reqwest::Method::POST,
1915                &path,
1916                RequestProtocol::ControlPlane,
1917                Some(&body),
1918            )
1919            .await
1920        {
1921            Ok(response) => response,
1922            Err(Error::Http { status, body }) => {
1923                return Err(Error::QueryFailed(query_failure(status, body)));
1924            }
1925            Err(error) => return Err(error),
1926        };
1927
1928        if let Some(envelope) = response
1929            .get("result_envelope")
1930            .filter(|envelope| !envelope.is_null())
1931        {
1932            return decode_wire_value(envelope, DEFAULT_CODEC);
1933        }
1934
1935        Ok(response.get("result").cloned().unwrap_or(Value::Null))
1936    }
1937
1938    /// Send a synchronous update using fixed Avro Value arguments.
1939    pub async fn update_workflow<T: Serialize>(
1940        &self,
1941        workflow_id: &str,
1942        update_name: &str,
1943        input: T,
1944        request_id: Option<&str>,
1945    ) -> Result<Value> {
1946        let response = self
1947            .update_workflow_response(workflow_id, update_name, input, request_id)
1948            .await?;
1949        if let Some(envelope) = response
1950            .get("result_envelope")
1951            .filter(|envelope| !envelope.is_null())
1952        {
1953            return decode_wire_value(envelope, DEFAULT_CODEC);
1954        }
1955        Ok(response.get("result").cloned().unwrap_or(response))
1956    }
1957
1958    /// Send a synchronous update and retain a bytes-capable Avro result.
1959    pub async fn update_workflow_avro_value<T: Serialize>(
1960        &self,
1961        workflow_id: &str,
1962        update_name: &str,
1963        input: T,
1964        request_id: Option<&str>,
1965    ) -> Result<AvroValue> {
1966        let response = self
1967            .update_workflow_response(workflow_id, update_name, input, request_id)
1968            .await?;
1969        let envelope = response
1970            .get("result_envelope")
1971            .filter(|envelope| !envelope.is_null())
1972            .ok_or_else(|| {
1973                Error::Codec(
1974                    "missing_payload_envelope: typed update result requires result_envelope"
1975                        .to_string(),
1976                )
1977            })?;
1978        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1979    }
1980
1981    async fn update_workflow_response<T: Serialize>(
1982        &self,
1983        workflow_id: &str,
1984        update_name: &str,
1985        input: T,
1986        request_id: Option<&str>,
1987    ) -> Result<Value> {
1988        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1989        let mut body = json!({
1990            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1991            "wait_for": "completed",
1992        });
1993        if let Some(request_id) = request_id {
1994            body["request_id"] = json!(request_id);
1995        }
1996        self.request_json(
1997            reqwest::Method::POST,
1998            &format!("/workflows/{workflow_id}/update/{update_name}"),
1999            RequestProtocol::ControlPlane,
2000            Some(&body),
2001        )
2002        .await
2003    }
2004
2005    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2006        let path = format!("/workflows/{workflow_id}");
2007        let mut data: WorkflowDescription = self
2008            .request_json(
2009                reqwest::Method::GET,
2010                &path,
2011                RequestProtocol::ControlPlane,
2012                Option::<&Value>::None,
2013            )
2014            .await?;
2015        data.decode_payloads()?;
2016        Ok(data)
2017    }
2018
2019    /// Describe one selected run, including historical terminal runs.
2020    pub async fn describe_workflow_run(
2021        &self,
2022        workflow_id: &str,
2023        run_id: &str,
2024    ) -> Result<WorkflowDescription> {
2025        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2026        let mut data: WorkflowDescription = self
2027            .request_json(
2028                reqwest::Method::GET,
2029                &path,
2030                RequestProtocol::ControlPlane,
2031                Option::<&Value>::None,
2032            )
2033            .await?;
2034        data.decode_payloads()?;
2035        Ok(data)
2036    }
2037
2038    pub async fn register_worker(
2039        &self,
2040        worker_id: &str,
2041        task_queue: &str,
2042        supported_workflow_types: Vec<String>,
2043        supported_activity_types: Vec<String>,
2044        max_concurrent_workflow_tasks: usize,
2045        max_concurrent_activity_tasks: usize,
2046    ) -> Result<RegisterWorkerResponse> {
2047        self.register_worker_with_capabilities(
2048            worker_id,
2049            task_queue,
2050            supported_workflow_types,
2051            supported_activity_types,
2052            max_concurrent_workflow_tasks,
2053            max_concurrent_activity_tasks,
2054            Vec::new(),
2055        )
2056        .await
2057    }
2058
2059    /// Register a worker and explicitly advertise additive worker capabilities.
2060    pub async fn register_worker_with_capabilities(
2061        &self,
2062        worker_id: &str,
2063        task_queue: &str,
2064        supported_workflow_types: Vec<String>,
2065        supported_activity_types: Vec<String>,
2066        max_concurrent_workflow_tasks: usize,
2067        max_concurrent_activity_tasks: usize,
2068        capabilities: Vec<String>,
2069    ) -> Result<RegisterWorkerResponse> {
2070        self.register_worker_with_command_contracts(
2071            worker_id,
2072            task_queue,
2073            supported_workflow_types,
2074            supported_activity_types,
2075            max_concurrent_workflow_tasks,
2076            max_concurrent_activity_tasks,
2077            capabilities,
2078            Value::Object(serde_json::Map::new()),
2079        )
2080        .await
2081    }
2082
2083    /// Register a worker and advertise its named query and update handlers.
2084    #[allow(clippy::too_many_arguments)]
2085    pub async fn register_worker_with_command_contracts(
2086        &self,
2087        worker_id: &str,
2088        task_queue: &str,
2089        supported_workflow_types: Vec<String>,
2090        supported_activity_types: Vec<String>,
2091        max_concurrent_workflow_tasks: usize,
2092        max_concurrent_activity_tasks: usize,
2093        capabilities: Vec<String>,
2094        workflow_command_contracts: Value,
2095    ) -> Result<RegisterWorkerResponse> {
2096        let mut body = json!({
2097            "worker_id": worker_id,
2098            "task_queue": task_queue,
2099            "runtime": "rust",
2100            "sdk_version": SDK_VERSION,
2101            "supported_workflow_types": supported_workflow_types,
2102            "supported_activity_types": supported_activity_types,
2103            "capabilities": capabilities,
2104            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2105            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2106        });
2107        if workflow_command_contracts
2108            .as_object()
2109            .is_some_and(|contracts| !contracts.is_empty())
2110        {
2111            body["workflow_command_contracts"] = workflow_command_contracts;
2112        }
2113
2114        self.request_json(
2115            reqwest::Method::POST,
2116            "/worker/register",
2117            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2118            Some(&body),
2119        )
2120        .await
2121    }
2122
2123    /// Gracefully remove one worker's registration through the worker plane.
2124    ///
2125    /// This operation is separate from operator-facing worker management. It
2126    /// uses worker-protocol authentication and returns the server's lease
2127    /// recovery result.
2128    pub async fn deregister_worker_registration(
2129        &self,
2130        worker_id: &str,
2131    ) -> Result<WorkerDeregistrationEnvelope> {
2132        let path = format!(
2133            "/worker/registrations/{}",
2134            percent_encode_path_segment(worker_id)
2135        );
2136        self.request_json(
2137            reqwest::Method::DELETE,
2138            &path,
2139            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2140            Option::<&Value>::None,
2141        )
2142        .await
2143    }
2144
2145    /// Long-poll for an ephemeral, read-only workflow query task.
2146    pub async fn poll_query_task(
2147        &self,
2148        worker_id: &str,
2149        task_queue: &str,
2150        timeout: Duration,
2151    ) -> Result<Option<QueryTask>> {
2152        Ok(self
2153            .poll_query_task_response(worker_id, task_queue, timeout)
2154            .await?
2155            .task)
2156    }
2157
2158    /// Poll a query task while preserving server stop and drain metadata.
2159    pub async fn poll_query_task_response(
2160        &self,
2161        worker_id: &str,
2162        task_queue: &str,
2163        timeout: Duration,
2164    ) -> Result<PollQueryTaskResponse> {
2165        let poll_request_id = unique_request_id("rust-query-poll");
2166        self.poll_query_task_response_with_request_id(
2167            worker_id,
2168            task_queue,
2169            timeout,
2170            &poll_request_id,
2171            1,
2172        )
2173        .await
2174    }
2175
2176    async fn poll_query_task_response_with_request_id(
2177        &self,
2178        worker_id: &str,
2179        task_queue: &str,
2180        timeout: Duration,
2181        poll_request_id: &str,
2182        transport_retries: usize,
2183    ) -> Result<PollQueryTaskResponse> {
2184        let timeout_seconds = long_poll_timeout_seconds(timeout);
2185        let body = json!({
2186            "worker_id": worker_id,
2187            "task_queue": task_queue,
2188            "poll_request_id": poll_request_id,
2189            "timeout_seconds": timeout_seconds,
2190        });
2191        self.poll_request_json(
2192            "/worker/query-tasks/poll",
2193            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2194            &body,
2195            timeout + Duration::from_secs(5),
2196            transport_retries,
2197        )
2198        .await
2199    }
2200
2201    /// Complete a query task without appending workflow history.
2202    pub async fn complete_query_task<T: Serialize>(
2203        &self,
2204        query_task_id: &str,
2205        lease_owner: &str,
2206        query_task_attempt: u64,
2207        result: T,
2208        codec: &str,
2209    ) -> Result<Value> {
2210        let typed_result = AvroValue::from_serialize(&result)?;
2211        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2212        self.complete_query_task_with_envelope(
2213            query_task_id,
2214            lease_owner,
2215            query_task_attempt,
2216            typed_result.into_json()?,
2217            result_envelope,
2218        )
2219        .await
2220    }
2221
2222    async fn complete_query_task_with_envelope(
2223        &self,
2224        query_task_id: &str,
2225        lease_owner: &str,
2226        query_task_attempt: u64,
2227        result: Value,
2228        result_envelope: Value,
2229    ) -> Result<Value> {
2230        let body = json!({
2231            "lease_owner": lease_owner,
2232            "query_task_attempt": query_task_attempt,
2233            "result": result,
2234            "result_envelope": result_envelope,
2235        });
2236        let path = format!("/worker/query-tasks/{query_task_id}/complete");
2237        let response = self
2238            .request_json(
2239                reqwest::Method::POST,
2240                &path,
2241                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2242                Some(&body),
2243            )
2244            .await;
2245        query_task_response(response)
2246    }
2247
2248    /// Report a stable machine-readable query-task failure.
2249    pub async fn fail_query_task(
2250        &self,
2251        query_task_id: &str,
2252        lease_owner: &str,
2253        query_task_attempt: u64,
2254        message: impl Into<String>,
2255        reason: impl Into<String>,
2256        failure_type: impl Into<String>,
2257    ) -> Result<Value> {
2258        let body = json!({
2259            "lease_owner": lease_owner,
2260            "query_task_attempt": query_task_attempt,
2261            "failure": {
2262                "message": message.into(),
2263                "reason": reason.into(),
2264                "type": failure_type.into(),
2265            }
2266        });
2267        let path = format!("/worker/query-tasks/{query_task_id}/fail");
2268        let response = self
2269            .request_json(
2270                reqwest::Method::POST,
2271                &path,
2272                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2273                Some(&body),
2274            )
2275            .await;
2276        query_task_response(response)
2277    }
2278
2279    pub async fn heartbeat_worker(
2280        &self,
2281        worker_id: &str,
2282        workflow_available: usize,
2283        activity_available: usize,
2284    ) -> Result<Value> {
2285        let body = json!({
2286            "worker_id": worker_id,
2287            "task_slots": {
2288                "workflow_available": workflow_available,
2289                "activity_available": activity_available
2290            },
2291            "process_metrics": {
2292                "process_id": std::process::id(),
2293                "process_uptime_seconds": 0
2294            }
2295        });
2296
2297        self.request_json(
2298            reqwest::Method::POST,
2299            "/worker/heartbeat",
2300            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2301            Some(&body),
2302        )
2303        .await
2304    }
2305
2306    pub async fn poll_workflow_task(
2307        &self,
2308        worker_id: &str,
2309        task_queue: &str,
2310        timeout: Duration,
2311    ) -> Result<Option<WorkflowTask>> {
2312        Ok(self
2313            .poll_workflow_task_response(worker_id, task_queue, timeout)
2314            .await?
2315            .task)
2316    }
2317
2318    pub async fn poll_workflow_task_response(
2319        &self,
2320        worker_id: &str,
2321        task_queue: &str,
2322        timeout: Duration,
2323    ) -> Result<PollWorkflowTaskResponse> {
2324        let poll_request_id = unique_request_id("rust-workflow-poll");
2325        self.poll_workflow_task_response_with_request_id(
2326            worker_id,
2327            task_queue,
2328            timeout,
2329            &poll_request_id,
2330            1,
2331        )
2332        .await
2333    }
2334
2335    async fn poll_workflow_task_response_with_request_id(
2336        &self,
2337        worker_id: &str,
2338        task_queue: &str,
2339        timeout: Duration,
2340        poll_request_id: &str,
2341        transport_retries: usize,
2342    ) -> Result<PollWorkflowTaskResponse> {
2343        let body = json!({
2344            "worker_id": worker_id,
2345            "task_queue": task_queue,
2346            "poll_request_id": poll_request_id,
2347            "timeout_seconds": long_poll_timeout_seconds(timeout),
2348        });
2349        let mut data: PollWorkflowTaskResponse = self
2350            .poll_request_json(
2351                "/worker/workflow-tasks/poll",
2352                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2353                &body,
2354                timeout + Duration::from_secs(5),
2355                transport_retries,
2356            )
2357            .await?;
2358
2359        if let Some(task) = data.task.as_mut() {
2360            self.fetch_remaining_workflow_history(worker_id, task)
2361                .await?;
2362        }
2363
2364        Ok(data)
2365    }
2366
2367    async fn fetch_remaining_workflow_history(
2368        &self,
2369        worker_id: &str,
2370        task: &mut WorkflowTask,
2371    ) -> Result<()> {
2372        let mut next_token = task.next_history_page_token.clone();
2373
2374        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2375            let lease_owner = task
2376                .lease_owner
2377                .clone()
2378                .unwrap_or_else(|| worker_id.to_string());
2379            let page = self
2380                .workflow_task_history_page(
2381                    &task.task_id,
2382                    &lease_owner,
2383                    task.workflow_task_attempt,
2384                    &token,
2385                )
2386                .await?;
2387
2388            task.append_history_page(page);
2389
2390            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2391                return Err(Error::Codec(
2392                    "workflow history pagination returned the same page token".to_string(),
2393                ));
2394            }
2395
2396            next_token = task.next_history_page_token.clone();
2397        }
2398
2399        Ok(())
2400    }
2401
2402    async fn workflow_task_history_page(
2403        &self,
2404        task_id: &str,
2405        lease_owner: &str,
2406        workflow_task_attempt: u64,
2407        next_history_page_token: &str,
2408    ) -> Result<WorkflowTaskHistoryPage> {
2409        let body = json!({
2410            "lease_owner": lease_owner,
2411            "workflow_task_attempt": workflow_task_attempt,
2412            "next_history_page_token": next_history_page_token
2413        });
2414        let path = format!("/worker/workflow-tasks/{task_id}/history");
2415
2416        self.request_json(
2417            reqwest::Method::POST,
2418            &path,
2419            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2420            Some(&body),
2421        )
2422        .await
2423    }
2424
2425    pub async fn complete_workflow_task(
2426        &self,
2427        task_id: &str,
2428        lease_owner: &str,
2429        workflow_task_attempt: u64,
2430        commands: Vec<Value>,
2431    ) -> Result<Value> {
2432        let body = json!({
2433            "lease_owner": lease_owner,
2434            "workflow_task_attempt": workflow_task_attempt,
2435            "commands": commands
2436        });
2437        let path = format!("/worker/workflow-tasks/{task_id}/complete");
2438        self.request_json(
2439            reqwest::Method::POST,
2440            &path,
2441            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2442            Some(&body),
2443        )
2444        .await
2445    }
2446
2447    pub async fn fail_workflow_task(
2448        &self,
2449        task_id: &str,
2450        lease_owner: &str,
2451        workflow_task_attempt: u64,
2452        message: impl Into<String>,
2453    ) -> Result<Value> {
2454        self.fail_workflow_task_with_type(
2455            task_id,
2456            lease_owner,
2457            workflow_task_attempt,
2458            message,
2459            "RustWorkflowTaskFailure",
2460        )
2461        .await
2462    }
2463
2464    async fn fail_workflow_task_with_type(
2465        &self,
2466        task_id: &str,
2467        lease_owner: &str,
2468        workflow_task_attempt: u64,
2469        message: impl Into<String>,
2470        failure_type: &str,
2471    ) -> Result<Value> {
2472        let body = json!({
2473            "lease_owner": lease_owner,
2474            "workflow_task_attempt": workflow_task_attempt,
2475            "failure": {
2476                "message": message.into(),
2477                "type": failure_type
2478            }
2479        });
2480        let path = format!("/worker/workflow-tasks/{task_id}/fail");
2481        self.request_json(
2482            reqwest::Method::POST,
2483            &path,
2484            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2485            Some(&body),
2486        )
2487        .await
2488    }
2489
2490    pub async fn poll_activity_task(
2491        &self,
2492        worker_id: &str,
2493        task_queue: &str,
2494        timeout: Duration,
2495    ) -> Result<Option<ActivityTask>> {
2496        Ok(self
2497            .poll_activity_task_response(worker_id, task_queue, timeout)
2498            .await?
2499            .task)
2500    }
2501
2502    /// Poll an activity task while preserving server stop and drain metadata.
2503    pub async fn poll_activity_task_response(
2504        &self,
2505        worker_id: &str,
2506        task_queue: &str,
2507        timeout: Duration,
2508    ) -> Result<PollActivityTaskResponse> {
2509        let poll_request_id = unique_request_id("rust-activity-poll");
2510        self.poll_activity_task_response_with_request_id(
2511            worker_id,
2512            task_queue,
2513            timeout,
2514            &poll_request_id,
2515            1,
2516        )
2517        .await
2518    }
2519
2520    async fn poll_activity_task_response_with_request_id(
2521        &self,
2522        worker_id: &str,
2523        task_queue: &str,
2524        timeout: Duration,
2525        poll_request_id: &str,
2526        transport_retries: usize,
2527    ) -> Result<PollActivityTaskResponse> {
2528        let body = json!({
2529            "worker_id": worker_id,
2530            "task_queue": task_queue,
2531            "poll_request_id": poll_request_id,
2532            "timeout_seconds": long_poll_timeout_seconds(timeout),
2533        });
2534        let data: PollActivityTaskResponse = self
2535            .poll_request_json(
2536                "/worker/activity-tasks/poll",
2537                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2538                &body,
2539                timeout + Duration::from_secs(5),
2540                transport_retries,
2541            )
2542            .await?;
2543        Ok(data)
2544    }
2545
2546    pub async fn complete_activity_task<T: Serialize>(
2547        &self,
2548        task_id: &str,
2549        activity_attempt_id: &str,
2550        lease_owner: &str,
2551        result: T,
2552        codec: &str,
2553    ) -> Result<Value> {
2554        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2555        let body = json!({
2556            "activity_attempt_id": activity_attempt_id,
2557            "lease_owner": lease_owner,
2558            "result": result
2559        });
2560        let path = format!("/worker/activity-tasks/{task_id}/complete");
2561        activity_task_response(
2562            self.request_json(
2563                reqwest::Method::POST,
2564                &path,
2565                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2566                Some(&body),
2567            )
2568            .await,
2569            "complete",
2570            task_id,
2571            activity_attempt_id,
2572        )
2573    }
2574
2575    pub async fn fail_activity_task(
2576        &self,
2577        task_id: &str,
2578        activity_attempt_id: &str,
2579        lease_owner: &str,
2580        message: impl Into<String>,
2581        non_retryable: bool,
2582    ) -> Result<Value> {
2583        let body = json!({
2584            "activity_attempt_id": activity_attempt_id,
2585            "lease_owner": lease_owner,
2586            "failure": {
2587                "message": message.into(),
2588                "type": "RustActivityFailure",
2589                "non_retryable": non_retryable
2590            }
2591        });
2592        let path = format!("/worker/activity-tasks/{task_id}/fail");
2593        activity_task_response(
2594            self.request_json(
2595                reqwest::Method::POST,
2596                &path,
2597                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2598                Some(&body),
2599            )
2600            .await,
2601            "fail",
2602            task_id,
2603            activity_attempt_id,
2604        )
2605    }
2606
2607    pub async fn heartbeat_activity_task<T: Serialize>(
2608        &self,
2609        task_id: &str,
2610        activity_attempt_id: &str,
2611        lease_owner: &str,
2612        details: T,
2613    ) -> Result<ActivityHeartbeatResponse> {
2614        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2615        let body = json!({
2616            "activity_attempt_id": activity_attempt_id,
2617            "lease_owner": lease_owner,
2618            "details": details
2619        });
2620        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2621        activity_task_response(
2622            self.request_json(
2623                reqwest::Method::POST,
2624                &path,
2625                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2626                Some(&body),
2627            )
2628            .await,
2629            "heartbeat",
2630            task_id,
2631            activity_attempt_id,
2632        )
2633    }
2634
2635    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2636        &self,
2637        method: reqwest::Method,
2638        path: &str,
2639        protocol: RequestProtocol,
2640        body: Option<&B>,
2641    ) -> Result<T> {
2642        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2643            .await
2644    }
2645
2646    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2647        &self,
2648        method: reqwest::Method,
2649        path: &str,
2650        protocol: RequestProtocol,
2651        body: Option<&B>,
2652        timeout: Duration,
2653    ) -> Result<T> {
2654        let auth_token = self.auth_token(protocol)?;
2655        let mut request = self
2656            .http
2657            .request(method, format!("{}/api{}", self.base_url, path))
2658            .timeout(timeout)
2659            .header(reqwest::header::ACCEPT, "application/json")
2660            .header(reqwest::header::CONTENT_TYPE, "application/json")
2661            .header("X-Namespace", &self.namespace);
2662
2663        match protocol {
2664            RequestProtocol::Worker(version) => {
2665                request = request.header("X-Durable-Workflow-Protocol-Version", version);
2666            }
2667            RequestProtocol::ControlPlane => {
2668                request = request.header(
2669                    "X-Durable-Workflow-Control-Plane-Version",
2670                    CONTROL_PLANE_VERSION,
2671                );
2672            }
2673        }
2674
2675        if let Some(token) = auth_token {
2676            request = request.bearer_auth(token);
2677        }
2678
2679        if let Some(body) = body {
2680            request = request.json(body);
2681        }
2682
2683        let response = request.send().await?;
2684        let status = response.status();
2685        let bytes = response.bytes().await?;
2686
2687        if !status.is_success() {
2688            let body = String::from_utf8_lossy(&bytes).to_string();
2689            if let Some(protocol) = protocol_failure(status, &body) {
2690                return Err(Error::Protocol(protocol));
2691            }
2692            return Err(Error::Http { status, body });
2693        }
2694
2695        if bytes.is_empty() {
2696            return Ok(serde_json::from_value(Value::Null)?);
2697        }
2698
2699        Ok(serde_json::from_slice(&bytes)?)
2700    }
2701
2702    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2703        &self,
2704        path: &str,
2705        protocol: RequestProtocol,
2706        body: &B,
2707        timeout: Duration,
2708        max_retries: usize,
2709    ) -> Result<T> {
2710        let mut retries = 0;
2711
2712        loop {
2713            let response = self
2714                .request_json_with_timeout(
2715                    reqwest::Method::POST,
2716                    path,
2717                    protocol,
2718                    Some(body),
2719                    timeout,
2720                )
2721                .await;
2722
2723            match response {
2724                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2725                response => return worker_poll_response(response),
2726            }
2727        }
2728    }
2729
2730    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2731        match protocol {
2732            RequestProtocol::Worker(_) => {
2733                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2734                    return Ok(Some(token));
2735                }
2736                if self.control_token.is_some() {
2737                    return Err(Error::MissingRoleCredentials {
2738                        role: "worker",
2739                        opposite_role: "control",
2740                    });
2741                }
2742                Ok(None)
2743            }
2744            RequestProtocol::ControlPlane => {
2745                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2746                    return Ok(Some(token));
2747                }
2748                if self.worker_token.is_some() {
2749                    return Err(Error::MissingRoleCredentials {
2750                        role: "control",
2751                        opposite_role: "worker",
2752                    });
2753                }
2754                Ok(None)
2755            }
2756        }
2757    }
2758}
2759
2760fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2761    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2762    let reason = body
2763        .get("reason")
2764        .and_then(Value::as_str)
2765        .unwrap_or("query_rejected")
2766        .to_string();
2767    let message = body
2768        .get("message")
2769        .or_else(|| body.get("error"))
2770        .and_then(Value::as_str)
2771        .unwrap_or("workflow query was rejected")
2772        .to_string();
2773
2774    QueryFailure {
2775        status: status.as_u16(),
2776        reason,
2777        message,
2778        body,
2779    }
2780}
2781
2782fn workflow_command_result(
2783    command: WorkflowCommandKind,
2784    data: Value,
2785    workflow_id: &str,
2786    run_id: Option<&str>,
2787) -> WorkflowCommandResult {
2788    WorkflowCommandResult {
2789        command,
2790        workflow_id: data
2791            .get("workflow_id")
2792            .and_then(Value::as_str)
2793            .unwrap_or(workflow_id)
2794            .to_string(),
2795        run_id: data
2796            .get("run_id")
2797            .and_then(Value::as_str)
2798            .or(run_id)
2799            .map(str::to_string),
2800        outcome: data
2801            .get("outcome")
2802            .and_then(Value::as_str)
2803            .map(str::to_string),
2804        reason: data
2805            .get("reason")
2806            .and_then(Value::as_str)
2807            .map(str::to_string),
2808        command_status: data
2809            .get("command_status")
2810            .and_then(Value::as_str)
2811            .map(str::to_string),
2812        raw: data,
2813    }
2814}
2815
2816fn workflow_command_rejection(
2817    command: WorkflowCommandKind,
2818    status: reqwest::StatusCode,
2819    raw_body: String,
2820    workflow_id: &str,
2821    run_id: Option<&str>,
2822) -> WorkflowCommandRejection {
2823    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2824    WorkflowCommandRejection {
2825        command,
2826        status: status.as_u16(),
2827        reason: body
2828            .get("reason")
2829            .and_then(Value::as_str)
2830            .unwrap_or("workflow_command_rejected")
2831            .to_string(),
2832        message: body
2833            .get("message")
2834            .or_else(|| body.get("error"))
2835            .and_then(Value::as_str)
2836            .unwrap_or("workflow lifecycle command was rejected")
2837            .to_string(),
2838        workflow_id: body
2839            .get("workflow_id")
2840            .and_then(Value::as_str)
2841            .unwrap_or(workflow_id)
2842            .to_string(),
2843        run_id: body
2844            .get("run_id")
2845            .and_then(Value::as_str)
2846            .or(run_id)
2847            .map(str::to_string),
2848        target_scope: body
2849            .get("target_scope")
2850            .and_then(Value::as_str)
2851            .map(str::to_string),
2852        body,
2853    }
2854}
2855
2856fn query_task_response(response: Result<Value>) -> Result<Value> {
2857    match response {
2858        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2859        response => response,
2860    }
2861}
2862
2863fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2864    match response {
2865        Err(Error::Http { status, body })
2866            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2867        {
2868            Ok(serde_json::from_str(&body)?)
2869        }
2870        response => response,
2871    }
2872}
2873
2874fn worker_poll_body_is_stop(body: &str) -> bool {
2875    serde_json::from_str::<Value>(body)
2876        .ok()
2877        .is_some_and(|body| {
2878            worker_poll_is_stop(
2879                body.get("poll_status").and_then(Value::as_str),
2880                body.get("reason").and_then(Value::as_str),
2881            )
2882        })
2883}
2884
2885fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2886    matches!(poll_status, Some("draining" | "stopped"))
2887        || matches!(reason, Some("worker_draining" | "worker_stopped"))
2888}
2889
2890fn query_task_rejection_is_final(error: &Error) -> bool {
2891    matches!(
2892        error,
2893        Error::QueryFailed(failure)
2894            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2895    )
2896}
2897
2898fn activity_task_response<T>(
2899    response: Result<T>,
2900    operation: &str,
2901    task_id: &str,
2902    activity_attempt_id: &str,
2903) -> Result<T> {
2904    match response {
2905        Err(Error::Http { status, body }) => {
2906            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2907            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2908                operation: operation.to_string(),
2909                status: status.as_u16(),
2910                reason: body
2911                    .get("reason")
2912                    .and_then(Value::as_str)
2913                    .unwrap_or("activity_task_rejected")
2914                    .to_string(),
2915                task_id: body
2916                    .get("task_id")
2917                    .and_then(Value::as_str)
2918                    .unwrap_or(task_id)
2919                    .to_string(),
2920                activity_attempt_id: body
2921                    .get("activity_attempt_id")
2922                    .and_then(Value::as_str)
2923                    .unwrap_or(activity_attempt_id)
2924                    .to_string(),
2925                cancel_requested: body
2926                    .get("cancel_requested")
2927                    .and_then(Value::as_bool)
2928                    .unwrap_or(false),
2929                can_continue: body.get("can_continue").and_then(Value::as_bool),
2930                run_closed_reason: body
2931                    .get("run_closed_reason")
2932                    .and_then(Value::as_str)
2933                    .map(str::to_string),
2934                body,
2935            }))
2936        }
2937        response => response,
2938    }
2939}
2940
2941fn activity_task_rejection_is_final(error: &Error) -> bool {
2942    matches!(
2943        error,
2944        Error::ActivityTaskRejected(rejection)
2945            if matches!(
2946                rejection.reason.as_str(),
2947                "run_cancelled"
2948                    | "run_terminated"
2949                    | "attempt_closed"
2950                    | "stale_attempt"
2951                    | "activity_cancelled"
2952                    | "task_cancelled"
2953                    | "run_closed"
2954                    | "activity_not_running"
2955                    | "attempt_not_found"
2956            )
2957    )
2958}
2959
2960fn workflow_task_completion_is_terminal_timeout(
2961    error: &Error,
2962    task_id: &str,
2963    workflow_task_attempt: u64,
2964    run_id: Option<&str>,
2965) -> bool {
2966    let Error::Http { status, body } = error else {
2967        return false;
2968    };
2969    if *status != reqwest::StatusCode::CONFLICT {
2970        return false;
2971    }
2972
2973    let Some(run_id) = run_id else {
2974        return false;
2975    };
2976    let Ok(body) = serde_json::from_str::<Value>(body) else {
2977        return false;
2978    };
2979
2980    body.get("recorded").and_then(Value::as_bool) == Some(false)
2981        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
2982        && body.get("run_status").and_then(Value::as_str) == Some("failed")
2983        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
2984        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
2985        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
2986}
2987
2988fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
2989    let body: Value = serde_json::from_str(raw_body).ok()?;
2990    let reason = body.get("reason")?.as_str()?;
2991    if !matches!(
2992        reason,
2993        "missing_protocol_version"
2994            | "unsupported_protocol_version"
2995            | "missing_control_plane_version"
2996            | "unsupported_control_plane_version"
2997    ) {
2998        return None;
2999    }
3000
3001    Some(ProtocolFailure {
3002        status: status.as_u16(),
3003        reason: reason.to_string(),
3004        message: body
3005            .get("message")
3006            .or_else(|| body.get("error"))
3007            .and_then(Value::as_str)
3008            .unwrap_or("protocol version rejected")
3009            .to_string(),
3010        supported_version: body
3011            .get("supported_version")
3012            .and_then(Value::as_str)
3013            .map(str::to_string),
3014        requested_version: body
3015            .get("requested_version")
3016            .and_then(Value::as_str)
3017            .map(str::to_string),
3018        body,
3019    })
3020}
3021
3022fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3023    timeout
3024        .as_secs()
3025        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3026        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3027}
3028
3029fn worker_operation_is_retryable(error: &Error) -> bool {
3030    match error {
3031        Error::Transport(error) => {
3032            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3033        }
3034        Error::Http { status, .. } => {
3035            matches!(
3036                *status,
3037                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3038            ) || status.is_server_error()
3039        }
3040        _ => false,
3041    }
3042}
3043
3044fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3045    let exponent = retry.saturating_sub(1).min(31) as u32;
3046    policy
3047        .initial_backoff
3048        .saturating_mul(1_u32 << exponent)
3049        .min(policy.max_backoff)
3050}
3051
3052#[derive(Debug)]
3053pub struct ClientBuilder {
3054    base_url: String,
3055    token: Option<String>,
3056    control_token: Option<String>,
3057    worker_token: Option<String>,
3058    namespace: String,
3059    timeout: Duration,
3060}
3061
3062impl ClientBuilder {
3063    pub fn token(mut self, token: Option<String>) -> Self {
3064        self.token = token;
3065        self
3066    }
3067
3068    pub fn control_token(mut self, token: Option<String>) -> Self {
3069        self.control_token = token;
3070        self
3071    }
3072
3073    pub fn worker_token(mut self, token: Option<String>) -> Self {
3074        self.worker_token = token;
3075        self
3076    }
3077
3078    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3079        self.namespace = namespace.into();
3080        self
3081    }
3082
3083    pub fn timeout(mut self, timeout: Duration) -> Self {
3084        self.timeout = timeout;
3085        self
3086    }
3087
3088    pub fn build(self) -> Result<Client> {
3089        let base_url = self.base_url.trim_end_matches('/').to_string();
3090        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3091            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3092            .unwrap_or_else(|_| base_url.ends_with("/api"));
3093
3094        if has_sdk_api_suffix {
3095            return Err(Error::InvalidBaseUrl);
3096        }
3097
3098        Ok(Client {
3099            http: reqwest::Client::builder().timeout(self.timeout).build()?,
3100            base_url,
3101            token: self.token,
3102            control_token: self.control_token,
3103            worker_token: self.worker_token,
3104            namespace: self.namespace,
3105        })
3106    }
3107}
3108
3109#[derive(Clone, Debug)]
3110pub struct WorkflowHandle {
3111    client: Client,
3112    pub workflow_id: String,
3113    pub run_id: Option<String>,
3114    pub workflow_type: String,
3115}
3116
3117impl WorkflowHandle {
3118    /// Describe whichever run is current for this stable workflow instance.
3119    pub async fn describe(&self) -> Result<WorkflowDescription> {
3120        self.client.describe_workflow(&self.workflow_id).await
3121    }
3122
3123    /// Describe the run identity originally selected by this handle.
3124    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3125        let run_id = self.run_id.as_deref().ok_or_else(|| {
3126            Error::Codec("run_id is required for selected-run description".to_string())
3127        })?;
3128        self.client
3129            .describe_workflow_run(&self.workflow_id, run_id)
3130            .await
3131    }
3132
3133    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3134        self.client
3135            .signal_workflow(&self.workflow_id, signal_name, input)
3136            .await
3137    }
3138
3139    /// Signal only if this handle's selected run is still current.
3140    pub async fn signal_selected_run<T: Serialize>(
3141        &self,
3142        signal_name: &str,
3143        input: T,
3144    ) -> Result<Value> {
3145        let run_id = self.run_id.as_deref().ok_or_else(|| {
3146            Error::Codec("run_id is required for selected-run signaling".to_string())
3147        })?;
3148        self.client
3149            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3150            .await
3151    }
3152
3153    /// Request cooperative cancellation of whichever run is current.
3154    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3155        self.client
3156            .cancel_workflow(&self.workflow_id, options)
3157            .await
3158    }
3159
3160    /// Request cancellation only if this handle's selected run is still current.
3161    pub async fn cancel_selected_run(
3162        &self,
3163        options: WorkflowCommandOptions,
3164    ) -> Result<WorkflowCommandResult> {
3165        let run_id = self.run_id.as_deref().ok_or_else(|| {
3166            Error::Codec("run_id is required for selected-run cancellation".to_string())
3167        })?;
3168        self.client
3169            .cancel_workflow_run(&self.workflow_id, run_id, options)
3170            .await
3171    }
3172
3173    /// Forcefully terminate whichever run is current.
3174    pub async fn terminate(
3175        &self,
3176        options: WorkflowCommandOptions,
3177    ) -> Result<WorkflowCommandResult> {
3178        self.client
3179            .terminate_workflow(&self.workflow_id, options)
3180            .await
3181    }
3182
3183    /// Terminate only if this handle's selected run is still current.
3184    pub async fn terminate_selected_run(
3185        &self,
3186        options: WorkflowCommandOptions,
3187    ) -> Result<WorkflowCommandResult> {
3188        let run_id = self.run_id.as_deref().ok_or_else(|| {
3189            Error::Codec("run_id is required for selected-run termination".to_string())
3190        })?;
3191        self.client
3192            .terminate_workflow_run(&self.workflow_id, run_id, options)
3193            .await
3194    }
3195
3196    /// Execute a named, read-only query against this workflow.
3197    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3198        self.client
3199            .query_workflow(&self.workflow_id, query_name, input)
3200            .await
3201    }
3202
3203    pub async fn query_avro_value<T: Serialize>(
3204        &self,
3205        query_name: &str,
3206        input: T,
3207    ) -> Result<AvroValue> {
3208        self.client
3209            .query_workflow_avro_value(&self.workflow_id, query_name, input)
3210            .await
3211    }
3212
3213    pub async fn update<T: Serialize>(
3214        &self,
3215        update_name: &str,
3216        input: T,
3217        request_id: Option<&str>,
3218    ) -> Result<Value> {
3219        self.client
3220            .update_workflow(&self.workflow_id, update_name, input, request_id)
3221            .await
3222    }
3223
3224    pub async fn update_avro_value<T: Serialize>(
3225        &self,
3226        update_name: &str,
3227        input: T,
3228        request_id: Option<&str>,
3229    ) -> Result<AvroValue> {
3230        self.client
3231            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3232            .await
3233    }
3234
3235    /// Query only if this handle's selected run is still current.
3236    pub async fn query_selected_run<T: Serialize>(
3237        &self,
3238        query_name: &str,
3239        input: T,
3240    ) -> Result<Value> {
3241        let run_id = self
3242            .run_id
3243            .as_deref()
3244            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3245        self.client
3246            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3247            .await
3248    }
3249
3250    /// Await the final terminal outcome of the current continue-as-new chain.
3251    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3252        self.result_target(options, None).await
3253    }
3254
3255    /// Await the final result without projecting Avro bytes through JSON.
3256    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3257        self.result_avro_value_target(options, None).await
3258    }
3259
3260    /// Await only the run identity originally selected by this handle.
3261    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3262        let run_id = self.run_id.as_deref().ok_or_else(|| {
3263            Error::Codec("run_id is required for selected-run result".to_string())
3264        })?;
3265        self.result_target(options, Some(run_id)).await
3266    }
3267
3268    /// Await the selected run's result on the lossless Avro Value surface.
3269    pub async fn result_selected_run_avro_value(
3270        &self,
3271        options: WorkflowResultOptions,
3272    ) -> Result<AvroValue> {
3273        let run_id = self.run_id.as_deref().ok_or_else(|| {
3274            Error::Codec("run_id is required for selected-run result".to_string())
3275        })?;
3276        self.result_avro_value_target(options, Some(run_id)).await
3277    }
3278
3279    async fn result_avro_value_target(
3280        &self,
3281        options: WorkflowResultOptions,
3282        selected_run_id: Option<&str>,
3283    ) -> Result<AvroValue> {
3284        let started = Instant::now();
3285
3286        loop {
3287            let description = match selected_run_id {
3288                Some(run_id) => {
3289                    self.client
3290                        .describe_workflow_run(&self.workflow_id, run_id)
3291                        .await?
3292                }
3293                None => self.describe().await?,
3294            };
3295            if description.is_completed() {
3296                return description.output_avro_value.ok_or_else(|| {
3297                    Error::Codec(
3298                        "missing_payload_envelope: typed workflow result requires output_envelope"
3299                            .to_string(),
3300                    )
3301                });
3302            }
3303            if description.is_terminal() {
3304                let outcome =
3305                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3306                return Err(match outcome.kind {
3307                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3308                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3309                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3310                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3311                });
3312            }
3313            if started.elapsed() >= options.timeout {
3314                return Err(Error::Timeout);
3315            }
3316            tokio::time::sleep(options.poll_interval).await;
3317        }
3318    }
3319
3320    async fn result_target(
3321        &self,
3322        options: WorkflowResultOptions,
3323        selected_run_id: Option<&str>,
3324    ) -> Result<Value> {
3325        let started = Instant::now();
3326
3327        loop {
3328            let description = match selected_run_id {
3329                Some(run_id) => {
3330                    self.client
3331                        .describe_workflow_run(&self.workflow_id, run_id)
3332                        .await?
3333                }
3334                None => self.describe().await?,
3335            };
3336            if description.is_completed() {
3337                return Ok(description.output.unwrap_or(Value::Null));
3338            }
3339
3340            if description.is_terminal() {
3341                let outcome =
3342                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3343                return Err(match outcome.kind {
3344                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3345                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3346                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3347                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3348                });
3349            }
3350
3351            if started.elapsed() >= options.timeout {
3352                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3353                    kind: WorkflowTerminalKind::TimedOut,
3354                    workflow_id: description
3355                        .workflow_id
3356                        .clone()
3357                        .unwrap_or_else(|| self.workflow_id.clone()),
3358                    run_id: description
3359                        .run_id
3360                        .clone()
3361                        .or_else(|| selected_run_id.map(str::to_string)),
3362                    reason: "result_wait_timeout".to_string(),
3363                    failure_category: Some("client_timeout".to_string()),
3364                    failure_id: None,
3365                    exception_type: None,
3366                    exception_class: None,
3367                    non_retryable: None,
3368                    message: Some(format!(
3369                        "workflow result was not terminal within {:?}",
3370                        options.timeout
3371                    )),
3372                    exception: None,
3373                    raw: description.raw_value(),
3374                }));
3375            }
3376
3377            tokio::time::sleep(options.poll_interval).await;
3378        }
3379    }
3380}
3381
3382#[derive(Clone, Copy, Debug)]
3383pub struct WorkflowResultOptions {
3384    pub poll_interval: Duration,
3385    pub timeout: Duration,
3386}
3387
3388impl Default for WorkflowResultOptions {
3389    fn default() -> Self {
3390        Self {
3391            poll_interval: Duration::from_millis(500),
3392            timeout: Duration::from_secs(30),
3393        }
3394    }
3395}
3396
3397#[derive(Clone, Debug, Deserialize)]
3398pub struct WorkflowDescription {
3399    pub workflow_id: Option<String>,
3400    pub run_id: Option<String>,
3401    pub workflow_type: Option<String>,
3402    pub status: Option<String>,
3403    #[serde(default)]
3404    pub closed_reason: Option<String>,
3405    #[serde(default)]
3406    pub error: Option<String>,
3407    #[serde(default)]
3408    pub failure: Option<Value>,
3409    #[serde(default)]
3410    pub exception: Option<Value>,
3411    #[serde(default)]
3412    pub failures: Vec<Value>,
3413    #[serde(default)]
3414    pub output: Option<Value>,
3415    #[serde(default)]
3416    pub output_envelope: Option<Value>,
3417    #[serde(skip)]
3418    pub output_avro_value: Option<AvroValue>,
3419    #[serde(flatten)]
3420    pub raw: HashMap<String, Value>,
3421}
3422
3423impl WorkflowDescription {
3424    pub fn is_completed(&self) -> bool {
3425        matches!(self.status.as_deref(), Some("completed" | "Completed"))
3426    }
3427
3428    pub fn is_terminal(&self) -> bool {
3429        matches!(
3430            self.status.as_deref(),
3431            Some(
3432                "completed"
3433                    | "Completed"
3434                    | "failed"
3435                    | "Failed"
3436                    | "cancelled"
3437                    | "Cancelled"
3438                    | "terminated"
3439                    | "Terminated"
3440                    | "timed_out"
3441                    | "TimedOut",
3442            )
3443        )
3444    }
3445
3446    fn decode_payloads(&mut self) -> Result<()> {
3447        if let Some(envelope) = &self.output_envelope {
3448            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3449            self.output = Some(value.clone().into_json()?);
3450            self.output_avro_value = Some(value);
3451        }
3452
3453        Ok(())
3454    }
3455
3456    fn raw_value(&self) -> Value {
3457        let mut data = self.raw.clone();
3458        data.insert(
3459            "workflow_id".to_string(),
3460            self.workflow_id
3461                .clone()
3462                .map(Value::String)
3463                .unwrap_or(Value::Null),
3464        );
3465        data.insert(
3466            "run_id".to_string(),
3467            self.run_id
3468                .clone()
3469                .map(Value::String)
3470                .unwrap_or(Value::Null),
3471        );
3472        data.insert(
3473            "workflow_type".to_string(),
3474            self.workflow_type
3475                .clone()
3476                .map(Value::String)
3477                .unwrap_or(Value::Null),
3478        );
3479        data.insert(
3480            "status".to_string(),
3481            self.status
3482                .clone()
3483                .map(Value::String)
3484                .unwrap_or(Value::Null),
3485        );
3486        data.insert(
3487            "closed_reason".to_string(),
3488            self.closed_reason
3489                .clone()
3490                .map(Value::String)
3491                .unwrap_or(Value::Null),
3492        );
3493        if let Some(failure) = &self.failure {
3494            data.insert("failure".to_string(), failure.clone());
3495        }
3496        if let Some(exception) = &self.exception {
3497            data.insert("exception".to_string(), exception.clone());
3498        }
3499        Value::Object(data.into_iter().collect())
3500    }
3501}
3502
3503fn workflow_terminal_outcome(
3504    description: &WorkflowDescription,
3505    workflow_id: &str,
3506    run_id: Option<&str>,
3507) -> WorkflowTerminalOutcome {
3508    let terminal_kind = description
3509        .closed_reason
3510        .as_deref()
3511        .or(description.status.as_deref())
3512        .unwrap_or("failed")
3513        .to_ascii_lowercase();
3514    let kind = match terminal_kind.as_str() {
3515        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3516        "terminated" => WorkflowTerminalKind::Terminated,
3517        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3518        _ => WorkflowTerminalKind::Failed,
3519    };
3520    let default_reason = match kind {
3521        WorkflowTerminalKind::Failed => "workflow_failed",
3522        WorkflowTerminalKind::Cancelled => "cancelled",
3523        WorkflowTerminalKind::Terminated => "terminated",
3524        WorkflowTerminalKind::TimedOut => "timed_out",
3525    };
3526    let failure = description
3527        .failure
3528        .as_ref()
3529        .filter(|value| value.is_object());
3530    let nested_failure = failure
3531        .and_then(|value| value.get("failures"))
3532        .and_then(Value::as_array)
3533        .and_then(|failures| failures.last())
3534        .or_else(|| description.failures.last());
3535    let exception = description
3536        .exception
3537        .clone()
3538        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3539        .or_else(|| {
3540            nested_failure
3541                .and_then(|value| value.get("exception_payload"))
3542                .cloned()
3543        });
3544    let string_field = |name: &str| {
3545        failure
3546            .and_then(|value| value.get(name))
3547            .and_then(Value::as_str)
3548            .or_else(|| {
3549                nested_failure
3550                    .and_then(|value| value.get(name))
3551                    .and_then(Value::as_str)
3552            })
3553            .map(str::to_string)
3554    };
3555    let exception_field = |name: &str| {
3556        exception
3557            .as_ref()
3558            .and_then(|value| value.get(name))
3559            .and_then(Value::as_str)
3560            .map(str::to_string)
3561    };
3562    let message = description
3563        .error
3564        .clone()
3565        .or_else(|| string_field("message"))
3566        .or_else(|| exception_field("message"));
3567    let reason = description
3568        .raw
3569        .get("reason")
3570        .and_then(Value::as_str)
3571        .map(str::to_string)
3572        .or_else(|| {
3573            failure
3574                .and_then(|value| value.get("reason"))
3575                .and_then(Value::as_str)
3576                .map(str::to_string)
3577        })
3578        .or_else(|| description.closed_reason.clone())
3579        .unwrap_or_else(|| default_reason.to_string());
3580    let failure_id = string_field("failure_id").or_else(|| {
3581        nested_failure
3582            .and_then(|value| value.get("id"))
3583            .and_then(Value::as_str)
3584            .map(str::to_string)
3585    });
3586
3587    WorkflowTerminalOutcome {
3588        kind,
3589        workflow_id: description
3590            .workflow_id
3591            .clone()
3592            .unwrap_or_else(|| workflow_id.to_string()),
3593        run_id: description
3594            .run_id
3595            .clone()
3596            .or_else(|| run_id.map(str::to_string)),
3597        reason,
3598        failure_category: string_field("failure_category")
3599            .or_else(|| Some(default_reason.to_string())),
3600        failure_id,
3601        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3602        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3603        non_retryable: failure
3604            .and_then(|value| value.get("non_retryable"))
3605            .and_then(Value::as_bool)
3606            .or_else(|| {
3607                nested_failure
3608                    .and_then(|value| value.get("non_retryable"))
3609                    .and_then(Value::as_bool)
3610            }),
3611        message,
3612        exception,
3613        raw: description.raw_value(),
3614    }
3615}
3616
3617#[derive(Clone, Debug, Deserialize)]
3618pub struct RegisterWorkerResponse {
3619    pub worker_id: String,
3620    pub registered: bool,
3621    #[serde(default)]
3622    pub heartbeat_interval_seconds: Option<u64>,
3623    #[serde(default)]
3624    pub protocol_version: Option<String>,
3625    #[serde(default)]
3626    pub server_capabilities: Option<Value>,
3627}
3628
3629/// Result of gracefully removing a worker-plane registration.
3630#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3631pub struct WorkerDeregistrationEnvelope {
3632    pub worker_id: String,
3633    pub outcome: String,
3634    pub recovered_workflow_task_count: u64,
3635}
3636
3637#[derive(Clone, Debug, Deserialize)]
3638pub struct PollWorkflowTaskResponse {
3639    #[serde(default)]
3640    pub task: Option<WorkflowTask>,
3641    #[serde(default)]
3642    pub poll_status: Option<String>,
3643    #[serde(default)]
3644    pub reason: Option<String>,
3645    #[serde(default)]
3646    pub protocol_version: Option<String>,
3647    #[serde(default)]
3648    pub server_capabilities: Option<Value>,
3649}
3650
3651impl PollWorkflowTaskResponse {
3652    /// Classify this response without parsing server display text.
3653    pub fn outcome(&self) -> WorkerPollOutcome {
3654        worker_poll_outcome(
3655            self.task.is_some(),
3656            self.poll_status.as_deref(),
3657            self.reason.as_deref(),
3658        )
3659    }
3660}
3661
3662#[derive(Clone, Debug, Deserialize)]
3663pub struct PollActivityTaskResponse {
3664    #[serde(default)]
3665    pub task: Option<ActivityTask>,
3666    #[serde(default)]
3667    pub poll_status: Option<String>,
3668    #[serde(default)]
3669    pub reason: Option<String>,
3670}
3671
3672impl PollActivityTaskResponse {
3673    /// Classify this response without parsing server display text.
3674    pub fn outcome(&self) -> WorkerPollOutcome {
3675        worker_poll_outcome(
3676            self.task.is_some(),
3677            self.poll_status.as_deref(),
3678            self.reason.as_deref(),
3679        )
3680    }
3681}
3682
3683#[derive(Clone, Debug, Deserialize)]
3684pub struct PollQueryTaskResponse {
3685    #[serde(default)]
3686    pub task: Option<QueryTask>,
3687    #[serde(default)]
3688    pub poll_status: Option<String>,
3689    #[serde(default)]
3690    pub reason: Option<String>,
3691}
3692
3693impl PollQueryTaskResponse {
3694    /// Classify this response without parsing server display text.
3695    pub fn outcome(&self) -> WorkerPollOutcome {
3696        worker_poll_outcome(
3697            self.task.is_some(),
3698            self.poll_status.as_deref(),
3699            self.reason.as_deref(),
3700        )
3701    }
3702}
3703
3704/// Stable classification for worker poll responses.
3705#[derive(Clone, Debug, PartialEq, Eq)]
3706pub enum WorkerPollOutcome {
3707    /// A task was leased and is available on the response.
3708    Task,
3709    /// No task was leased, but the worker should continue polling.
3710    Idle {
3711        poll_status: Option<String>,
3712        reason: Option<String>,
3713    },
3714    /// The server asked this worker to stop claiming new work.
3715    Stop {
3716        poll_status: Option<String>,
3717        reason: Option<String>,
3718    },
3719}
3720
3721impl WorkerPollOutcome {
3722    pub fn should_stop(&self) -> bool {
3723        matches!(self, Self::Stop { .. })
3724    }
3725}
3726
3727fn worker_poll_outcome(
3728    has_task: bool,
3729    poll_status: Option<&str>,
3730    reason: Option<&str>,
3731) -> WorkerPollOutcome {
3732    if worker_poll_is_stop(poll_status, reason) {
3733        return WorkerPollOutcome::Stop {
3734            poll_status: poll_status.map(str::to_string),
3735            reason: reason.map(str::to_string),
3736        };
3737    }
3738
3739    if has_task {
3740        WorkerPollOutcome::Task
3741    } else {
3742        WorkerPollOutcome::Idle {
3743            poll_status: poll_status.map(str::to_string),
3744            reason: reason.map(str::to_string),
3745        }
3746    }
3747}
3748
3749/// An ephemeral server-routed query task.
3750#[derive(Clone, Debug, Deserialize)]
3751pub struct QueryTask {
3752    pub query_task_id: String,
3753    #[serde(default = "default_workflow_task_attempt")]
3754    pub query_task_attempt: u64,
3755    #[serde(default)]
3756    pub lease_owner: Option<String>,
3757    #[serde(default)]
3758    pub workflow_id: Option<String>,
3759    #[serde(default)]
3760    pub run_id: Option<String>,
3761    pub workflow_type: String,
3762    pub query_name: String,
3763    #[serde(default = "default_payload_codec")]
3764    pub payload_codec: String,
3765    #[serde(default)]
3766    pub workflow_arguments: Option<Value>,
3767    #[serde(default)]
3768    pub query_arguments: Option<Value>,
3769    #[serde(default)]
3770    pub history_events: Vec<HistoryEvent>,
3771    #[serde(default)]
3772    pub history_export: Option<Value>,
3773    #[serde(default)]
3774    pub run_status: Option<String>,
3775}
3776
3777#[derive(Clone, Debug, Deserialize)]
3778pub struct WorkflowTask {
3779    pub task_id: String,
3780    #[serde(default)]
3781    pub workflow_id: Option<String>,
3782    #[serde(default)]
3783    pub run_id: Option<String>,
3784    pub workflow_type: String,
3785    #[serde(default = "default_payload_codec")]
3786    pub payload_codec: String,
3787    #[serde(default)]
3788    pub arguments: Option<Value>,
3789    #[serde(default)]
3790    pub history_events: Vec<HistoryEvent>,
3791    #[serde(default)]
3792    pub total_history_events: Option<u64>,
3793    #[serde(default)]
3794    pub history_size_bytes: Option<u64>,
3795    #[serde(default)]
3796    pub continue_as_new_recommended: Option<bool>,
3797    #[serde(default)]
3798    pub history_budget_pressure: Option<String>,
3799    #[serde(default)]
3800    pub next_history_page_token: Option<String>,
3801    #[serde(default = "default_workflow_task_attempt")]
3802    pub workflow_task_attempt: u64,
3803    #[serde(default)]
3804    pub workflow_signal_id: Option<String>,
3805    #[serde(default)]
3806    pub signal_name: Option<String>,
3807    #[serde(default)]
3808    pub signal_arguments: Option<Value>,
3809    #[serde(default)]
3810    pub workflow_update_id: Option<String>,
3811    #[serde(default)]
3812    pub update_name: Option<String>,
3813    #[serde(default)]
3814    pub lease_owner: Option<String>,
3815}
3816
3817impl WorkflowTask {
3818    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3819        self.history_events.extend(page.history_events);
3820
3821        if page.total_history_events.is_some() {
3822            self.total_history_events = page.total_history_events;
3823        }
3824
3825        self.next_history_page_token = page
3826            .next_history_page_token
3827            .filter(|token| !token.is_empty());
3828    }
3829}
3830
3831#[derive(Clone, Debug, Deserialize)]
3832struct WorkflowTaskHistoryPage {
3833    #[serde(default)]
3834    history_events: Vec<HistoryEvent>,
3835    #[serde(default)]
3836    total_history_events: Option<u64>,
3837    #[serde(default)]
3838    next_history_page_token: Option<String>,
3839}
3840
3841#[derive(Clone, Debug, Deserialize)]
3842pub struct ActivityTask {
3843    pub task_id: String,
3844    #[serde(default)]
3845    pub activity_attempt_id: Option<String>,
3846    #[serde(default)]
3847    pub attempt_id: Option<String>,
3848    pub activity_type: String,
3849    #[serde(default = "default_payload_codec")]
3850    pub payload_codec: String,
3851    #[serde(default)]
3852    pub arguments: Option<Value>,
3853    #[serde(default = "default_attempt_number")]
3854    pub attempt_number: u64,
3855    #[serde(default)]
3856    pub lease_owner: Option<String>,
3857}
3858
3859#[derive(Clone, Debug, Deserialize)]
3860pub struct HistoryEvent {
3861    #[serde(alias = "type")]
3862    pub event_type: String,
3863    #[serde(default)]
3864    pub payload: Value,
3865    #[serde(flatten)]
3866    pub raw: HashMap<String, Value>,
3867}
3868
3869/// One decoded signal in the committed workflow-history snapshot.
3870#[derive(Clone, Debug, PartialEq)]
3871pub struct QuerySignal {
3872    pub id: Option<String>,
3873    pub name: String,
3874    pub arguments: Vec<Value>,
3875    avro_arguments: Vec<AvroValue>,
3876    pub workflow_sequence: Option<u64>,
3877}
3878
3879impl QuerySignal {
3880    /// Lossless fixed Avro Value arguments for this committed signal.
3881    pub fn arguments_avro_value(&self) -> &[AvroValue] {
3882        &self.avro_arguments
3883    }
3884}
3885
3886/// Immutable state supplied to a registered query handler.
3887///
3888/// This context intentionally exposes no activity, signal-wait, or command
3889/// APIs. Query handlers inspect committed history and return a value; query
3890/// completion does not append an event or advance deterministic execution.
3891#[derive(Clone, Debug)]
3892pub struct QueryContext {
3893    pub workflow_id: Option<String>,
3894    pub run_id: Option<String>,
3895    pub workflow_type: String,
3896    pub run_status: Option<String>,
3897    workflow_input: Value,
3898    workflow_input_avro_value: AvroValue,
3899    history_events: Arc<Vec<HistoryEvent>>,
3900    signal_events: Arc<Vec<QuerySignal>>,
3901}
3902
3903impl QueryContext {
3904    /// The normalized argument list used to start the workflow.
3905    pub fn workflow_input(&self) -> &Value {
3906        &self.workflow_input
3907    }
3908
3909    /// The lossless fixed Avro Value argument list used to start the workflow.
3910    pub fn workflow_input_avro_value(&self) -> &AvroValue {
3911        &self.workflow_input_avro_value
3912    }
3913
3914    /// The immutable committed history used for this query snapshot.
3915    pub fn history_events(&self) -> &[HistoryEvent] {
3916        self.history_events.as_slice()
3917    }
3918
3919    /// All decoded signals in committed workflow order.
3920    pub fn signal_events(&self) -> &[QuerySignal] {
3921        self.signal_events.as_slice()
3922    }
3923
3924    /// Decoded argument lists for each committed signal with `signal_name`.
3925    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3926        self.signal_events
3927            .iter()
3928            .filter(|signal| signal.name == signal_name)
3929            .map(|signal| signal.arguments.clone())
3930            .collect()
3931    }
3932
3933    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
3934    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3935        self.signal_events
3936            .iter()
3937            .filter(|signal| signal.name == signal_name)
3938            .map(|signal| signal.avro_arguments.clone())
3939            .collect()
3940    }
3941}
3942
3943#[derive(Clone, Debug, Deserialize)]
3944pub struct ActivityHeartbeatResponse {
3945    #[serde(default)]
3946    pub cancel_requested: bool,
3947    #[serde(default)]
3948    pub heartbeat_recorded: bool,
3949    #[serde(default)]
3950    pub can_continue: Option<bool>,
3951    #[serde(default)]
3952    pub reason: Option<String>,
3953    #[serde(default)]
3954    pub run_closed_reason: Option<String>,
3955    #[serde(default)]
3956    pub run_closed_at: Option<String>,
3957    #[serde(default)]
3958    pub lease_expires_at: Option<String>,
3959    #[serde(default)]
3960    pub last_heartbeat_at: Option<String>,
3961}
3962
3963impl ActivityHeartbeatResponse {
3964    /// Whether the activity should stop instead of attempting completion.
3965    pub fn should_stop(&self) -> bool {
3966        self.cancel_requested || self.can_continue == Some(false)
3967    }
3968}
3969
3970fn default_payload_codec() -> String {
3971    DEFAULT_CODEC.to_string()
3972}
3973
3974fn default_workflow_task_attempt() -> u64 {
3975    1
3976}
3977
3978fn default_attempt_number() -> u64 {
3979    1
3980}
3981
3982type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3983type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
3984type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
3985type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
3986type ReplayedWorkflowHandler =
3987    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
3988type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3989type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
3990type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
3991type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3992type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
3993type ReplayedQueryHandler = Arc<
3994    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
3995        + Send
3996        + Sync,
3997>;
3998type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
3999
4000struct ReplayedWorkflowInvocation {
4001    future: WorkflowFuture,
4002    snapshot: WorkflowStateSnapshot,
4003}
4004
4005#[derive(Clone)]
4006struct RegisteredWorkflow {
4007    execute: WorkflowHandler,
4008    replay: Option<ReplayedWorkflowHandler>,
4009    state_type: Option<TypeId>,
4010}
4011
4012#[derive(Clone)]
4013enum RegisteredQuery {
4014    Snapshot(QueryHandler),
4015    Replayed {
4016        state_type: TypeId,
4017        handler: ReplayedQueryHandler,
4018    },
4019}
4020
4021#[derive(Clone, Debug)]
4022pub struct WorkerHeartbeatObservation {
4023    pub worker_id: String,
4024    pub task_queue: String,
4025    pub acknowledged_at_unix_millis: u64,
4026    pub acknowledgement: Value,
4027}
4028
4029/// Bounded retry policy for worker poll acquisition and worker heartbeats.
4030///
4031/// Expected empty long polls are normal successful responses. Transport
4032/// failures, HTTP 408/429 responses, and server errors are retried with capped
4033/// exponential backoff. Authentication, protocol, codec, and handler failures
4034/// are never retried by the worker.
4035#[derive(Clone, Copy, Debug)]
4036pub struct WorkerRetryPolicy {
4037    /// Number of retries after the initial request fails.
4038    pub max_retries: usize,
4039    /// Delay before the first retry.
4040    pub initial_backoff: Duration,
4041    /// Maximum delay between retries.
4042    pub max_backoff: Duration,
4043}
4044
4045impl Default for WorkerRetryPolicy {
4046    fn default() -> Self {
4047        Self {
4048            max_retries: 5,
4049            initial_backoff: Duration::from_millis(100),
4050            max_backoff: Duration::from_secs(5),
4051        }
4052    }
4053}
4054
4055#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4056enum ManagedPollOutcome {
4057    Idle,
4058    Handled,
4059    Stop,
4060}
4061
4062#[derive(Clone)]
4063pub struct Worker {
4064    client: Client,
4065    worker_id: String,
4066    task_queue: String,
4067    workflows: HashMap<String, RegisteredWorkflow>,
4068    activities: HashMap<String, ActivityHandler>,
4069    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4070    updates: HashMap<String, HashMap<String, UpdateHandler>>,
4071    max_concurrent_workflow_tasks: usize,
4072    max_concurrent_activity_tasks: usize,
4073    poll_timeout: Duration,
4074    heartbeat_interval: Duration,
4075    retry_policy: WorkerRetryPolicy,
4076    heartbeat_observer: Option<WorkerHeartbeatObserver>,
4077}
4078
4079impl Worker {
4080    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4081        Self {
4082            client,
4083            worker_id: default_worker_id(),
4084            task_queue: task_queue.into(),
4085            workflows: HashMap::new(),
4086            activities: HashMap::new(),
4087            queries: HashMap::new(),
4088            updates: HashMap::new(),
4089            max_concurrent_workflow_tasks: 10,
4090            max_concurrent_activity_tasks: 10,
4091            poll_timeout: Duration::from_secs(30),
4092            heartbeat_interval: Duration::from_secs(60),
4093            retry_policy: WorkerRetryPolicy::default(),
4094            heartbeat_observer: None,
4095        }
4096    }
4097
4098    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4099        self.worker_id = worker_id.into();
4100        self
4101    }
4102
4103    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4104        self.poll_timeout = timeout;
4105        self
4106    }
4107
4108    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4109        self.heartbeat_interval = interval;
4110        self
4111    }
4112
4113    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
4114    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4115        self.retry_policy = policy;
4116        self
4117    }
4118
4119    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4120    where
4121        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4122    {
4123        self.heartbeat_observer = Some(Arc::new(observer));
4124        self
4125    }
4126
4127    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4128        self.max_concurrent_workflow_tasks = count.max(1);
4129        self
4130    }
4131
4132    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4133        self.max_concurrent_activity_tasks = count.max(1);
4134        self
4135    }
4136
4137    /// Register a workflow handler.
4138    ///
4139    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
4140    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
4141    /// while acquiring or decoding a worker task remain worker-operation
4142    /// failures and do not get converted into workflow outcomes.
4143    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4144    where
4145        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4146        Fut: Future<Output = Result<Value>> + Send + 'static,
4147    {
4148        let handler = Arc::new(handler);
4149        self.workflows.insert(
4150            workflow_type.into(),
4151            RegisteredWorkflow {
4152                execute: Arc::new(move |ctx, input| {
4153                    let handler = Arc::clone(&handler);
4154                    Box::pin(async move {
4155                        let result = handler(ctx, input.into_json()?).await?;
4156                        AvroValue::from_serialize(&result)
4157                    })
4158                }),
4159                replay: None,
4160                state_type: None,
4161            },
4162        );
4163    }
4164
4165    /// Register a workflow on the lossless fixed Avro Value surface.
4166    pub fn register_workflow_avro_value<F, Fut>(
4167        &mut self,
4168        workflow_type: impl Into<String>,
4169        handler: F,
4170    ) where
4171        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4172        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4173    {
4174        self.workflows.insert(
4175            workflow_type.into(),
4176            RegisteredWorkflow {
4177                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4178                replay: None,
4179                state_type: None,
4180            },
4181        );
4182    }
4183
4184    /// Register a workflow whose typed instance state can be reconstructed for queries.
4185    ///
4186    /// `state_factory` creates a fresh instance for every normal workflow task and
4187    /// query replay. The workflow handler is the single source of truth for state
4188    /// transitions: it updates [`WorkflowInstance`] after activities and signals
4189    /// resolve. Query replay runs this same handler over committed history and
4190    /// discards any commands it would emit.
4191    pub fn register_replayed_workflow<S, Factory, F, Fut>(
4192        &mut self,
4193        workflow_type: impl Into<String>,
4194        state_factory: Factory,
4195        handler: F,
4196    ) where
4197        S: Clone + Send + Sync + 'static,
4198        Factory: Fn() -> S + Send + Sync + 'static,
4199        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4200        Fut: Future<Output = Result<Value>> + Send + 'static,
4201    {
4202        let state_factory = Arc::new(state_factory);
4203        let handler = Arc::new(handler);
4204
4205        let execute_factory = Arc::clone(&state_factory);
4206        let execute_handler = Arc::clone(&handler);
4207        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4208            let state = WorkflowInstance::new(execute_factory());
4209            let handler = Arc::clone(&execute_handler);
4210            Box::pin(async move {
4211                let result = handler(ctx, input.into_json()?, state).await?;
4212                AvroValue::from_serialize(&result)
4213            }) as WorkflowFuture
4214        });
4215
4216        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4217            let state = WorkflowInstance::new(state_factory());
4218            let snapshot_state = state.clone();
4219            let snapshot: WorkflowStateSnapshot =
4220                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4221            let replay_handler = Arc::clone(&handler);
4222            let future = async move {
4223                let result = replay_handler(ctx, input.into_json()?, state).await?;
4224                AvroValue::from_serialize(&result)
4225            };
4226            ReplayedWorkflowInvocation {
4227                future: Box::pin(future),
4228                snapshot,
4229            }
4230        });
4231
4232        self.workflows.insert(
4233            workflow_type.into(),
4234            RegisteredWorkflow {
4235                execute,
4236                replay: Some(replay),
4237                state_type: Some(TypeId::of::<S>()),
4238            },
4239        );
4240    }
4241
4242    /// Register a replayable workflow on the lossless fixed Avro Value surface.
4243    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4244        &mut self,
4245        workflow_type: impl Into<String>,
4246        state_factory: Factory,
4247        handler: F,
4248    ) where
4249        S: Clone + Send + Sync + 'static,
4250        Factory: Fn() -> S + Send + Sync + 'static,
4251        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4252        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4253    {
4254        let state_factory = Arc::new(state_factory);
4255        let handler = Arc::new(handler);
4256
4257        let execute_factory = Arc::clone(&state_factory);
4258        let execute_handler = Arc::clone(&handler);
4259        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4260            let state = WorkflowInstance::new(execute_factory());
4261            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4262        });
4263
4264        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4265            let state = WorkflowInstance::new(state_factory());
4266            let snapshot_state = state.clone();
4267            let snapshot: WorkflowStateSnapshot =
4268                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4269            ReplayedWorkflowInvocation {
4270                future: Box::pin(handler(ctx, input, state)),
4271                snapshot,
4272            }
4273        });
4274
4275        self.workflows.insert(
4276            workflow_type.into(),
4277            RegisteredWorkflow {
4278                execute,
4279                replay: Some(replay),
4280                state_type: Some(TypeId::of::<S>()),
4281            },
4282        );
4283    }
4284
4285    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4286    where
4287        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4288        Fut: Future<Output = Result<Value>> + Send + 'static,
4289    {
4290        let handler = Arc::new(handler);
4291        self.activities.insert(
4292            activity_type.into(),
4293            Arc::new(move |ctx, args| {
4294                let handler = Arc::clone(&handler);
4295                Box::pin(async move {
4296                    let result = handler(ctx, args.into_json()?).await?;
4297                    AvroValue::from_serialize(&result)
4298                })
4299            }),
4300        );
4301    }
4302
4303    /// Register an activity on the lossless fixed Avro Value surface.
4304    pub fn register_activity_avro_value<F, Fut>(
4305        &mut self,
4306        activity_type: impl Into<String>,
4307        handler: F,
4308    ) where
4309        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4310        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4311    {
4312        self.activities.insert(
4313            activity_type.into(),
4314            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4315        );
4316    }
4317
4318    /// Register a named, read-only query handler for a workflow type.
4319    ///
4320    /// The workflow type must also be registered with [`Worker::register_workflow`]
4321    /// before the worker runs. The handler receives only an immutable committed
4322    /// state snapshot and normalized query arguments.
4323    pub fn register_query<F, Fut>(
4324        &mut self,
4325        workflow_type: impl Into<String>,
4326        query_name: impl Into<String>,
4327        handler: F,
4328    ) where
4329        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4330        Fut: Future<Output = Result<Value>> + Send + 'static,
4331    {
4332        let handler = Arc::new(handler);
4333        self.queries
4334            .entry(workflow_type.into())
4335            .or_default()
4336            .insert(
4337                query_name.into(),
4338                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4339                    let handler = Arc::clone(&handler);
4340                    Box::pin(async move {
4341                        let result = handler(ctx, args.into_json()?).await?;
4342                        AvroValue::from_serialize(&result)
4343                    })
4344                })),
4345            );
4346    }
4347
4348    /// Register a query handler on the lossless fixed Avro Value surface.
4349    pub fn register_query_avro_value<F, Fut>(
4350        &mut self,
4351        workflow_type: impl Into<String>,
4352        query_name: impl Into<String>,
4353        handler: F,
4354    ) where
4355        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4356        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4357    {
4358        self.queries
4359            .entry(workflow_type.into())
4360            .or_default()
4361            .insert(
4362                query_name.into(),
4363                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4364            );
4365    }
4366
4367    /// Register a named query against deterministically replayed instance state.
4368    ///
4369    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
4370    /// same state type `S`. The handler receives an immutable, detached state
4371    /// clone, so successful and failed queries cannot affect workflow execution
4372    /// or the state reconstructed by a later query.
4373    pub fn register_replayed_query<S, F, Fut>(
4374        &mut self,
4375        workflow_type: impl Into<String>,
4376        query_name: impl Into<String>,
4377        handler: F,
4378    ) where
4379        S: Clone + Send + Sync + 'static,
4380        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4381        Fut: Future<Output = Result<Value>> + Send + 'static,
4382    {
4383        let handler = Arc::new(handler);
4384        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4385            let state = state.downcast::<S>().map_err(|_| {
4386                "registered query state type does not match the replayed workflow state".to_string()
4387            })?;
4388            let handler = Arc::clone(&handler);
4389            Ok(Box::pin(async move {
4390                let result = handler(ctx, state, args.into_json()?).await?;
4391                AvroValue::from_serialize(&result)
4392            }))
4393        });
4394
4395        self.queries
4396            .entry(workflow_type.into())
4397            .or_default()
4398            .insert(
4399                query_name.into(),
4400                RegisteredQuery::Replayed {
4401                    state_type: TypeId::of::<S>(),
4402                    handler: erased_handler,
4403                },
4404            );
4405    }
4406
4407    /// Register a replayed-state query on the lossless fixed Avro Value surface.
4408    pub fn register_replayed_query_avro_value<S, F, Fut>(
4409        &mut self,
4410        workflow_type: impl Into<String>,
4411        query_name: impl Into<String>,
4412        handler: F,
4413    ) where
4414        S: Clone + Send + Sync + 'static,
4415        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4416        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4417    {
4418        let handler = Arc::new(handler);
4419        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4420            let state = state.downcast::<S>().map_err(|_| {
4421                "registered query state type does not match the replayed workflow state".to_string()
4422            })?;
4423            Ok(Box::pin(handler(ctx, state, args)))
4424        });
4425
4426        self.queries
4427            .entry(workflow_type.into())
4428            .or_default()
4429            .insert(
4430                query_name.into(),
4431                RegisteredQuery::Replayed {
4432                    state_type: TypeId::of::<S>(),
4433                    handler: erased_handler,
4434                },
4435            );
4436    }
4437
4438    /// Register a synchronous workflow update handler.
4439    pub fn register_update<F, Fut>(
4440        &mut self,
4441        workflow_type: impl Into<String>,
4442        update_name: impl Into<String>,
4443        handler: F,
4444    ) where
4445        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4446        Fut: Future<Output = Result<Value>> + Send + 'static,
4447    {
4448        let handler = Arc::new(handler);
4449        self.updates
4450            .entry(workflow_type.into())
4451            .or_default()
4452            .insert(
4453                update_name.into(),
4454                Arc::new(move |ctx, args| {
4455                    let handler = Arc::clone(&handler);
4456                    Box::pin(async move {
4457                        let result = handler(ctx, args.into_json()?).await?;
4458                        AvroValue::from_serialize(&result)
4459                    })
4460                }),
4461            );
4462    }
4463
4464    /// Register an update handler on the lossless fixed Avro Value surface.
4465    pub fn register_update_avro_value<F, Fut>(
4466        &mut self,
4467        workflow_type: impl Into<String>,
4468        update_name: impl Into<String>,
4469        handler: F,
4470    ) where
4471        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4472        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4473    {
4474        self.updates
4475            .entry(workflow_type.into())
4476            .or_default()
4477            .insert(
4478                update_name.into(),
4479                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4480            );
4481    }
4482
4483    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4484        let mut command_contracts = serde_json::Map::new();
4485        for workflow_type in self.workflows.keys() {
4486            let mut queries = self
4487                .queries
4488                .get(workflow_type)
4489                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4490                .unwrap_or_default();
4491            queries.sort();
4492            let mut updates = self
4493                .updates
4494                .get(workflow_type)
4495                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4496                .unwrap_or_default();
4497            updates.sort();
4498            if !queries.is_empty() || !updates.is_empty() {
4499                command_contracts.insert(
4500                    workflow_type.clone(),
4501                    json!({
4502                        "queries": queries,
4503                        "updates": updates,
4504                    }),
4505                );
4506            }
4507        }
4508
4509        self.client
4510            .register_worker_with_command_contracts(
4511                &self.worker_id,
4512                &self.task_queue,
4513                self.workflows.keys().cloned().collect(),
4514                self.activities.keys().cloned().collect(),
4515                self.max_concurrent_workflow_tasks,
4516                self.max_concurrent_activity_tasks,
4517                [
4518                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4519                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4520                ]
4521                .into_iter()
4522                .flatten()
4523                .collect(),
4524                Value::Object(command_contracts),
4525            )
4526            .await
4527    }
4528
4529    /// Run until shutdown or a terminal worker error occurs.
4530    ///
4531    /// Empty long-poll expirations do not stop the worker. Retryable poll and
4532    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
4533    /// authentication, protocol, and other non-retryable failures are returned.
4534    pub async fn run(&self) -> Result<()> {
4535        self.run_until(std::future::pending::<()>()).await
4536    }
4537
4538    /// Run until `shutdown` resolves or a terminal worker error occurs.
4539    ///
4540    /// This has the same liveness and terminal-error contract as [`Worker::run`].
4541    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4542    where
4543        F: Future<Output = ()>,
4544    {
4545        let registration = self.register().await?;
4546        if !registration.registered {
4547            return Err(Error::WorkerLoop(format!(
4548                "worker registration for {:?} was not accepted",
4549                self.worker_id
4550            )));
4551        }
4552        let registered_worker_id = registration.worker_id.clone();
4553        let primary = self.run_registered_until(shutdown, registration).await;
4554        let deregistration = self
4555            .client
4556            .deregister_worker_registration(&registered_worker_id)
4557            .await;
4558
4559        match (primary, deregistration) {
4560            (Ok(()), Ok(_)) => Ok(()),
4561            (Ok(()), Err(deregistration)) => Err(deregistration),
4562            (Err(primary), Ok(_)) => Err(primary),
4563            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4564                primary: Box::new(primary),
4565                deregistration: Box::new(deregistration),
4566            }),
4567        }
4568    }
4569
4570    async fn run_registered_until<F>(
4571        &self,
4572        shutdown: F,
4573        registration: RegisterWorkerResponse,
4574    ) -> Result<()>
4575    where
4576        F: Future<Output = ()>,
4577    {
4578        let heartbeat_interval = Duration::from_secs(
4579            registration
4580                .heartbeat_interval_seconds
4581                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4582        );
4583        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
4584        // from the completion of the preceding attempt, including its bounded
4585        // retries. A fixed-epoch interval can leave an already-due tick queued
4586        // while an acknowledgement is slow, producing a catch-up heartbeat as
4587        // soon as that request completes.
4588        let heartbeat = tokio::time::sleep(Duration::ZERO);
4589        tokio::pin!(heartbeat);
4590        tokio::pin!(shutdown);
4591        let stop = Arc::new(AtomicBool::new(false));
4592        // Poll responses may already have leased server-side work by the time
4593        // they become ready, so each poller owns its responses through
4594        // completion or failure instead of racing raw polls in this select.
4595        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4596            let worker = self.clone();
4597            let stop = Arc::clone(&stop);
4598            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4599        });
4600        let mut activity_poller = (!self.activities.is_empty()).then(|| {
4601            let worker = self.clone();
4602            let stop = Arc::clone(&stop);
4603            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4604        });
4605        let mut query_poller = (!self.queries.is_empty()).then(|| {
4606            let worker = self.clone();
4607            let stop = Arc::clone(&stop);
4608            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4609        });
4610
4611        loop {
4612            tokio::select! {
4613                _ = &mut shutdown => {
4614                    stop.store(true, Ordering::SeqCst);
4615                    break;
4616                }
4617                _ = &mut heartbeat => {
4618                    let result = self.retry_worker_operation(|| {
4619                        self.client.heartbeat_worker(
4620                            &self.worker_id,
4621                            self.max_concurrent_workflow_tasks,
4622                            self.max_concurrent_activity_tasks,
4623                        )
4624                    }).await;
4625                    heartbeat
4626                        .as_mut()
4627                        .reset(tokio::time::Instant::now() + heartbeat_interval);
4628                    match result {
4629                        Ok(acknowledgement) => {
4630                            if let Some(observer) = &self.heartbeat_observer {
4631                                observer(&WorkerHeartbeatObservation {
4632                                    worker_id: self.worker_id.clone(),
4633                                    task_queue: self.task_queue.clone(),
4634                                    acknowledged_at_unix_millis: SystemTime::now()
4635                                        .duration_since(UNIX_EPOCH)
4636                                        .unwrap_or_default()
4637                                        .as_millis()
4638                                        .min(u64::MAX as u128)
4639                                        as u64,
4640                                    acknowledgement,
4641                                });
4642                            }
4643                        }
4644                        Err(error) => {
4645                            stop.store(true, Ordering::SeqCst);
4646                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4647                            return Err(error);
4648                        }
4649                    }
4650                }
4651                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4652                    workflow_poller = None;
4653                    let stopped_by_server = stop.load(Ordering::SeqCst);
4654                    stop.store(true, Ordering::SeqCst);
4655                    let poller_result = optional_poller_result("workflow", result);
4656                    let join_result =
4657                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4658                    poller_result?;
4659                    join_result?;
4660                    if stopped_by_server {
4661                        return Ok(());
4662                    }
4663                    return Err(Error::WorkerLoop(
4664                        "workflow poller stopped unexpectedly".to_string(),
4665                    ));
4666                }
4667                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4668                    activity_poller = None;
4669                    let stopped_by_server = stop.load(Ordering::SeqCst);
4670                    stop.store(true, Ordering::SeqCst);
4671                    let poller_result = optional_poller_result("activity", result);
4672                    let join_result =
4673                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4674                    poller_result?;
4675                    join_result?;
4676                    if stopped_by_server {
4677                        return Ok(());
4678                    }
4679                    return Err(Error::WorkerLoop(
4680                        "activity poller stopped unexpectedly".to_string(),
4681                    ));
4682                }
4683                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4684                    query_poller = None;
4685                    let stopped_by_server = stop.load(Ordering::SeqCst);
4686                    stop.store(true, Ordering::SeqCst);
4687                    let poller_result = optional_poller_result("query", result);
4688                    let join_result =
4689                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4690                    poller_result?;
4691                    join_result?;
4692                    if stopped_by_server {
4693                        return Ok(());
4694                    }
4695                    return Err(Error::WorkerLoop(
4696                        "query poller stopped unexpectedly".to_string(),
4697                    ));
4698                }
4699            }
4700        }
4701
4702        join_pollers(
4703            workflow_poller.take(),
4704            activity_poller.take(),
4705            query_poller.take(),
4706        )
4707        .await
4708    }
4709
4710    /// Poll and settle at most one task from each enabled task family.
4711    ///
4712    /// A workflow may reach its server-enforced run deadline while this worker
4713    /// holds a task. When the completion endpoint authoritatively rejects that
4714    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
4715    /// terminal `run_status=failed`, the workflow tick is considered settled:
4716    /// the late command was not recorded and cannot replace the terminal run.
4717    /// Every other completion rejection remains an error. This worker-level
4718    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
4719    /// only bounds how long a client waits for a result.
4720    ///
4721    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
4722    /// the original [`Error::Http`] status and response body.
4723    pub async fn run_once(&self) -> Result<usize> {
4724        let mut handled = 0;
4725        match self.poll_workflow_once().await? {
4726            ManagedPollOutcome::Handled => handled += 1,
4727            ManagedPollOutcome::Stop => return Ok(handled),
4728            ManagedPollOutcome::Idle => {}
4729        }
4730        match self.poll_activity_once().await? {
4731            ManagedPollOutcome::Handled => handled += 1,
4732            ManagedPollOutcome::Stop => return Ok(handled),
4733            ManagedPollOutcome::Idle => {}
4734        }
4735        if !self.queries.is_empty() {
4736            match self.poll_query_once().await? {
4737                ManagedPollOutcome::Handled => handled += 1,
4738                ManagedPollOutcome::Stop => return Ok(handled),
4739                ManagedPollOutcome::Idle => {}
4740            }
4741        }
4742        Ok(handled)
4743    }
4744
4745    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4746        let poll_request_id = unique_request_id("rust-workflow-poll");
4747        let response = self
4748            .retry_worker_operation(|| {
4749                self.client.poll_workflow_task_response_with_request_id(
4750                    &self.worker_id,
4751                    &self.task_queue,
4752                    self.poll_timeout,
4753                    &poll_request_id,
4754                    0,
4755                )
4756            })
4757            .await?;
4758        if response.outcome().should_stop() {
4759            return Ok(ManagedPollOutcome::Stop);
4760        }
4761        let Some(task) = response.task else {
4762            return Ok(ManagedPollOutcome::Idle);
4763        };
4764
4765        let task_id = task.task_id.clone();
4766        let attempt = task.workflow_task_attempt;
4767        let run_id = task.run_id.clone();
4768        let lease_owner = task
4769            .lease_owner
4770            .clone()
4771            .unwrap_or_else(|| self.worker_id.clone());
4772
4773        match self.execute_workflow_task(task) {
4774            Ok(commands) if commands.is_empty() => {
4775                // A replay can consume a recorded pending durable command
4776                // without producing a new command. The standalone protocol
4777                // acknowledges that state through the typed waiting outcome;
4778                // an empty completion is rejected by servers that require at
4779                // least one executable command.
4780                self.client
4781                    .fail_workflow_task_with_type(
4782                        &task_id,
4783                        &lease_owner,
4784                        attempt,
4785                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4786                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4787                    )
4788                    .await?;
4789            }
4790            Ok(commands) => {
4791                let completion = self
4792                    .client
4793                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4794                    .await;
4795                if let Err(error) = completion {
4796                    if !workflow_task_completion_is_terminal_timeout(
4797                        &error,
4798                        &task_id,
4799                        attempt,
4800                        run_id.as_deref(),
4801                    ) {
4802                        return Err(error);
4803                    }
4804                }
4805            }
4806            Err(error) => {
4807                self.client
4808                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4809                    .await?;
4810            }
4811        }
4812
4813        Ok(ManagedPollOutcome::Handled)
4814    }
4815
4816    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4817        while !stop.load(Ordering::SeqCst) {
4818            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4819                stop.store(true, Ordering::SeqCst);
4820                break;
4821            }
4822        }
4823
4824        Ok(())
4825    }
4826
4827    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4828        let poll_request_id = unique_request_id("rust-activity-poll");
4829        let response = self
4830            .retry_worker_operation(|| {
4831                self.client.poll_activity_task_response_with_request_id(
4832                    &self.worker_id,
4833                    &self.task_queue,
4834                    self.poll_timeout,
4835                    &poll_request_id,
4836                    0,
4837                )
4838            })
4839            .await?;
4840        if response.outcome().should_stop() {
4841            return Ok(ManagedPollOutcome::Stop);
4842        }
4843        let Some(task) = response.task else {
4844            return Ok(ManagedPollOutcome::Idle);
4845        };
4846
4847        let task_id = task.task_id.clone();
4848        let attempt_id = task
4849            .activity_attempt_id
4850            .clone()
4851            .or(task.attempt_id.clone())
4852            .unwrap_or_default();
4853        let lease_owner = task
4854            .lease_owner
4855            .clone()
4856            .unwrap_or_else(|| self.worker_id.clone());
4857        let codec = task.payload_codec.clone();
4858        let result = self.execute_activity_task(task).await;
4859        match result {
4860            Ok(value) => {
4861                let completion = self
4862                    .client
4863                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4864                    .await;
4865                if let Err(error) = completion {
4866                    if !activity_task_rejection_is_final(&error) {
4867                        return Err(error);
4868                    }
4869                }
4870            }
4871            Err(error) => {
4872                let failure = self
4873                    .client
4874                    .fail_activity_task(
4875                        &task_id,
4876                        &attempt_id,
4877                        &lease_owner,
4878                        error.to_string(),
4879                        false,
4880                    )
4881                    .await;
4882                if let Err(error) = failure {
4883                    if !activity_task_rejection_is_final(&error) {
4884                        return Err(error);
4885                    }
4886                }
4887            }
4888        }
4889
4890        Ok(ManagedPollOutcome::Handled)
4891    }
4892
4893    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4894        while !stop.load(Ordering::SeqCst) {
4895            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4896                stop.store(true, Ordering::SeqCst);
4897                break;
4898            }
4899        }
4900
4901        Ok(())
4902    }
4903
4904    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4905        let poll_request_id = unique_request_id("rust-query-poll");
4906        let response = self
4907            .retry_worker_operation(|| {
4908                self.client.poll_query_task_response_with_request_id(
4909                    &self.worker_id,
4910                    &self.task_queue,
4911                    self.poll_timeout,
4912                    &poll_request_id,
4913                    0,
4914                )
4915            })
4916            .await?;
4917        if response.outcome().should_stop() {
4918            return Ok(ManagedPollOutcome::Stop);
4919        }
4920        let Some(task) = response.task else {
4921            return Ok(ManagedPollOutcome::Idle);
4922        };
4923
4924        let query_task_id = task.query_task_id.clone();
4925        let attempt = task.query_task_attempt;
4926        let lease_owner = task
4927            .lease_owner
4928            .clone()
4929            .unwrap_or_else(|| self.worker_id.clone());
4930        let codec = task.payload_codec.clone();
4931
4932        match self.execute_query_task(task).await {
4933            Ok(value) => {
4934                let result_envelope = match encode_typed_envelope(&value, &codec) {
4935                    Ok(result_envelope) => result_envelope,
4936                    Err(error) => {
4937                        let failure = self
4938                            .client
4939                            .fail_query_task(
4940                                &query_task_id,
4941                                &lease_owner,
4942                                attempt,
4943                                error.to_string(),
4944                                "query_result_encode_failed",
4945                                "QueryResultEncodeFailed",
4946                            )
4947                            .await;
4948                        if let Err(error) = failure {
4949                            if !query_task_rejection_is_final(&error) {
4950                                return Err(error);
4951                            }
4952                        }
4953                        return Ok(ManagedPollOutcome::Handled);
4954                    }
4955                };
4956
4957                if let Err(error) = self
4958                    .client
4959                    .complete_query_task_with_envelope(
4960                        &query_task_id,
4961                        &lease_owner,
4962                        attempt,
4963                        value.clone().into_json()?,
4964                        result_envelope,
4965                    )
4966                    .await
4967                {
4968                    if !query_task_rejection_is_final(&error) {
4969                        return Err(error);
4970                    }
4971                }
4972            }
4973            Err(failure) => {
4974                let result = self
4975                    .client
4976                    .fail_query_task(
4977                        &query_task_id,
4978                        &lease_owner,
4979                        attempt,
4980                        failure.message,
4981                        failure.reason,
4982                        failure.failure_type,
4983                    )
4984                    .await;
4985                if let Err(error) = result {
4986                    if !query_task_rejection_is_final(&error) {
4987                        return Err(error);
4988                    }
4989                }
4990            }
4991        }
4992
4993        Ok(ManagedPollOutcome::Handled)
4994    }
4995
4996    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4997        while !stop.load(Ordering::SeqCst) {
4998            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
4999                stop.store(true, Ordering::SeqCst);
5000                break;
5001            }
5002        }
5003
5004        Ok(())
5005    }
5006
5007    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5008    where
5009        F: FnMut() -> Fut,
5010        Fut: Future<Output = Result<T>>,
5011    {
5012        let mut retries = 0;
5013
5014        loop {
5015            match operation().await {
5016                Err(error)
5017                    if worker_operation_is_retryable(&error)
5018                        && retries < self.retry_policy.max_retries =>
5019                {
5020                    retries += 1;
5021                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5022                }
5023                result => return result,
5024            }
5025        }
5026    }
5027
5028    async fn execute_query_task(
5029        &self,
5030        mut task: QueryTask,
5031    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5032        if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
5033            return Err(QueryTaskExecutionFailure::new(
5034                "query_payload_decode_failed",
5035                format!(
5036                    "cannot decode query payload with unsupported codec {:?}",
5037                    task.payload_codec
5038                ),
5039                "QueryPayloadDecodeFailed",
5040            ));
5041        }
5042
5043        if !self.workflows.contains_key(&task.workflow_type) {
5044            return Err(QueryTaskExecutionFailure::new(
5045                "query_workflow_type_not_registered",
5046                format!("no workflow registered for type {:?}", task.workflow_type),
5047                "WorkflowTypeNotRegistered",
5048            ));
5049        }
5050
5051        let Some(handlers) = self.queries.get(&task.workflow_type) else {
5052            return Err(QueryTaskExecutionFailure::new(
5053                "query_handler_unavailable",
5054                format!(
5055                    "query handlers are unavailable for workflow type {:?}",
5056                    task.workflow_type
5057                ),
5058                "QueryHandlerUnavailable",
5059            ));
5060        };
5061        let Some(query) = handlers.get(&task.query_name) else {
5062            return Err(QueryTaskExecutionFailure::new(
5063                "rejected_unknown_query",
5064                format!("unknown query {:?}", task.query_name),
5065                "QueryFailed",
5066            ));
5067        };
5068
5069        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5070            .map_err(|error| {
5071            QueryTaskExecutionFailure::new(
5072                "query_payload_decode_failed",
5073                format!("cannot decode query arguments: {error}"),
5074                "QueryPayloadDecodeFailed",
5075            )
5076        })?;
5077        let workflow_input_typed =
5078            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5079                .map_err(|error| {
5080                    QueryTaskExecutionFailure::new(
5081                        "query_workflow_state_unavailable",
5082                        format!("cannot decode workflow start input: {error}"),
5083                        "QueryWorkflowStateUnavailable",
5084                    )
5085                })?;
5086        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5087            QueryTaskExecutionFailure::new(
5088                "query_workflow_state_unavailable",
5089                format!("cannot project workflow start input: {error}"),
5090                "QueryWorkflowStateUnavailable",
5091            )
5092        })?;
5093        hydrate_query_history_from_export(&mut task).map_err(|error| {
5094            QueryTaskExecutionFailure::new(
5095                "query_workflow_state_unavailable",
5096                format!("cannot restore query history snapshot: {error}"),
5097                "QueryWorkflowStateUnavailable",
5098            )
5099        })?;
5100        enrich_query_history_from_export(&mut task).map_err(|error| {
5101            QueryTaskExecutionFailure::new(
5102                "query_workflow_state_unavailable",
5103                format!("cannot restore compact query history payloads: {error}"),
5104                "QueryWorkflowStateUnavailable",
5105            )
5106        })?;
5107        let signal_events = query_signal_events(&task).map_err(|error| {
5108            QueryTaskExecutionFailure::new(
5109                "query_workflow_state_unavailable",
5110                format!("cannot decode committed workflow signals: {error}"),
5111                "QueryWorkflowStateUnavailable",
5112            )
5113        })?;
5114        let history_events = Arc::new(std::mem::take(&mut task.history_events));
5115        let context = QueryContext {
5116            workflow_id: task.workflow_id,
5117            run_id: task.run_id,
5118            workflow_type: task.workflow_type.clone(),
5119            run_status: task.run_status,
5120            workflow_input,
5121            workflow_input_avro_value: workflow_input_typed.clone(),
5122            history_events: Arc::clone(&history_events),
5123            signal_events: Arc::new(signal_events),
5124        };
5125
5126        let future = match query {
5127            RegisteredQuery::Snapshot(handler) => handler(context, args),
5128            RegisteredQuery::Replayed {
5129                state_type,
5130                handler,
5131            } => {
5132                let workflow = self
5133                    .workflows
5134                    .get(&task.workflow_type)
5135                    .expect("workflow registration was checked above");
5136                if workflow.state_type != Some(*state_type) {
5137                    return Err(QueryTaskExecutionFailure::new(
5138                        "query_workflow_state_unavailable",
5139                        "replayed query state type does not match its workflow registration",
5140                        "QueryWorkflowStateUnavailable",
5141                    ));
5142                }
5143                let replay = workflow.replay.as_ref().ok_or_else(|| {
5144                    QueryTaskExecutionFailure::new(
5145                        "query_workflow_state_unavailable",
5146                        format!(
5147                            "workflow type {:?} is not registered for instance-state replay",
5148                            task.workflow_type
5149                        ),
5150                        "QueryWorkflowStateUnavailable",
5151                    )
5152                })?;
5153                let workflow_state = Arc::new(Mutex::new(
5154                    WorkflowState::new_with_identity(
5155                        history_events.as_ref().clone(),
5156                        context.workflow_id.clone(),
5157                        context.run_id.clone(),
5158                        self.task_queue.clone(),
5159                        task.payload_codec,
5160                        None,
5161                    )
5162                    .map_err(|error| {
5163                        QueryTaskExecutionFailure::new(
5164                            "query_workflow_state_unavailable",
5165                            format!("workflow replay failed before query: {error}"),
5166                            "QueryWorkflowStateUnavailable",
5167                        )
5168                    })?,
5169                ));
5170                let workflow_context = WorkflowContext {
5171                    state: workflow_state,
5172                };
5173                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5174                let mut cx = TaskContext::from_waker(noop_waker_ref());
5175                match invocation.future.as_mut().poll(&mut cx) {
5176                    Poll::Ready(Ok(_)) => {
5177                        workflow_context
5178                            .ensure_history_consumed()
5179                            .map_err(|error| {
5180                                QueryTaskExecutionFailure::new(
5181                                    "query_workflow_state_unavailable",
5182                                    format!("workflow replay failed before query: {error}"),
5183                                    "QueryWorkflowStateUnavailable",
5184                                )
5185                            })?;
5186                    }
5187                    Poll::Ready(Err(error)) => {
5188                        return Err(QueryTaskExecutionFailure::new(
5189                            "query_workflow_state_unavailable",
5190                            format!("workflow replay failed before query: {error}"),
5191                            "QueryWorkflowStateUnavailable",
5192                        ));
5193                    }
5194                    Poll::Pending => {
5195                        let commands = workflow_context.take_commands().map_err(|error| {
5196                            QueryTaskExecutionFailure::new(
5197                                "query_workflow_state_unavailable",
5198                                format!("workflow replay failed before query: {error}"),
5199                                "QueryWorkflowStateUnavailable",
5200                            )
5201                        })?;
5202                        if commands.is_empty()
5203                            && !workflow_context
5204                                .matched_recorded_pending()
5205                                .map_err(|error| {
5206                                    QueryTaskExecutionFailure::new(
5207                                        "query_workflow_state_unavailable",
5208                                        format!("workflow replay failed before query: {error}"),
5209                                        "QueryWorkflowStateUnavailable",
5210                                    )
5211                                })?
5212                        {
5213                            return Err(QueryTaskExecutionFailure::new(
5214                                "query_workflow_state_unavailable",
5215                                "workflow replay yielded without a durable command",
5216                                "QueryWorkflowStateUnavailable",
5217                            ));
5218                        }
5219                    }
5220                }
5221                let state = (invocation.snapshot)().map_err(|error| {
5222                    QueryTaskExecutionFailure::new(
5223                        "query_workflow_state_unavailable",
5224                        format!("cannot snapshot replayed workflow state: {error}"),
5225                        "QueryWorkflowStateUnavailable",
5226                    )
5227                })?;
5228                handler(context, state, args).map_err(|message| {
5229                    QueryTaskExecutionFailure::new(
5230                        "query_workflow_state_unavailable",
5231                        message,
5232                        "QueryWorkflowStateUnavailable",
5233                    )
5234                })?
5235            }
5236        };
5237
5238        future.await.map_err(|error| {
5239            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5240        })
5241    }
5242
5243    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5244        if let Some(update_id) = task
5245            .workflow_update_id
5246            .as_deref()
5247            .filter(|update_id| !update_id.is_empty())
5248        {
5249            return self.execute_update_task(&task, update_id);
5250        }
5251
5252        let workflow = self
5253            .workflows
5254            .get(&task.workflow_type)
5255            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5256        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5257        let resume_signal = decode_resume_signal(&task)?;
5258        let history_budget = WorkflowHistoryBudget {
5259            event_count: task
5260                .total_history_events
5261                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5262            size_bytes: task.history_size_bytes,
5263            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5264            pressure: task.history_budget_pressure.clone(),
5265        };
5266        let mut workflow_state = WorkflowState::new_with_identity(
5267            task.history_events,
5268            task.workflow_id,
5269            task.run_id,
5270            self.task_queue.clone(),
5271            task.payload_codec.clone(),
5272            resume_signal,
5273        )?;
5274        workflow_state.history_budget = history_budget;
5275        let state = Arc::new(Mutex::new(workflow_state));
5276        let ctx = WorkflowContext { state };
5277        let mut future = (workflow.execute)(ctx.clone(), input);
5278        let mut cx = TaskContext::from_waker(noop_waker_ref());
5279
5280        match future.as_mut().poll(&mut cx) {
5281            Poll::Ready(Ok(result)) => {
5282                ctx.ensure_history_consumed()?;
5283                let result = encode_typed_envelope(&result, &task.payload_codec)?;
5284                let mut commands = ctx.take_commands()?;
5285                commands.push(json!({
5286                    "type": "complete_workflow",
5287                    "result": result
5288                }));
5289                Ok(commands)
5290            }
5291            Poll::Ready(Err(error)) => {
5292                if let Error::ContinueAsNew(request) = error {
5293                    let mut commands = ctx.take_commands()?;
5294                    if let Some(command) = ctx.continue_as_new_command(request)? {
5295                        commands.push(command);
5296                    }
5297                    ctx.ensure_history_consumed()?;
5298                    return Ok(commands);
5299                }
5300                // A handler error must not hide a committed durable command that
5301                // upgraded workflow code no longer consumes.
5302                ctx.ensure_history_consumed()?;
5303                if workflow_task_integrity_error(&error) {
5304                    // Replay and protocol failures describe the workflow-task
5305                    // decision itself. Do not let commands queued earlier in
5306                    // this uncommitted decision escape alongside a terminal
5307                    // workflow failure.
5308                    return Err(error);
5309                }
5310                let mut commands = ctx.take_commands()?;
5311                commands.push(workflow_failure_command(&error));
5312                Ok(commands)
5313            }
5314            Poll::Pending => {
5315                let commands = ctx.take_commands()?;
5316                if commands.is_empty() && !ctx.matched_recorded_pending()? {
5317                    Err(Error::WorkflowYieldedWithoutCommand)
5318                } else {
5319                    Ok(commands)
5320                }
5321            }
5322        }
5323    }
5324
5325    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5326        if !self.workflows.contains_key(&task.workflow_type) {
5327            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5328        }
5329
5330        let accepted = task.history_events.iter().rev().find_map(|event| {
5331            (event.event_type == "UpdateAccepted"
5332                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5333            .then_some(&event.payload)
5334        });
5335        let update_name = accepted
5336            .and_then(|payload| payload.get("update_name"))
5337            .and_then(Value::as_str)
5338            .or(task.update_name.as_deref())
5339            .unwrap_or_default();
5340        let Some(handler) = self
5341            .updates
5342            .get(&task.workflow_type)
5343            .and_then(|handlers| handlers.get(update_name))
5344        else {
5345            return Ok(vec![json!({
5346                "type": "fail_update",
5347                "update_id": update_id,
5348                "message": format!(
5349                    "no update handler is registered for {}.{update_name}",
5350                    task.workflow_type
5351                ),
5352                "exception_type": "UnknownUpdate",
5353                "non_retryable": true,
5354            })]);
5355        };
5356        let arguments = accepted
5357            .and_then(|payload| payload.get("arguments"))
5358            .or(task.arguments.as_ref());
5359        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5360        let context = QueryContext {
5361            workflow_id: task.workflow_id.clone(),
5362            run_id: task.run_id.clone(),
5363            workflow_type: task.workflow_type.clone(),
5364            run_status: Some("running".to_string()),
5365            workflow_input: Value::Null,
5366            workflow_input_avro_value: AvroValue::Null,
5367            history_events: Arc::new(task.history_events.clone()),
5368            signal_events: Arc::new(Vec::new()),
5369        };
5370        let mut future = handler(context, arguments);
5371        let mut cx = TaskContext::from_waker(noop_waker_ref());
5372
5373        match future.as_mut().poll(&mut cx) {
5374            Poll::Ready(Ok(result)) => Ok(vec![json!({
5375                "type": "complete_update",
5376                "update_id": update_id,
5377                "result": encode_typed_envelope(&result, &task.payload_codec)?,
5378            })]),
5379            Poll::Ready(Err(error)) => Ok(vec![json!({
5380                "type": "fail_update",
5381                "update_id": update_id,
5382                "message": error.to_string(),
5383                "exception_type": "UpdateFailed",
5384                "non_retryable": true,
5385            })]),
5386            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5387        }
5388    }
5389
5390    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5391        let handler = self
5392            .activities
5393            .get(&task.activity_type)
5394            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5395        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5396        let attempt_id = task
5397            .activity_attempt_id
5398            .clone()
5399            .or(task.attempt_id.clone())
5400            .unwrap_or_default();
5401        let lease_owner = task
5402            .lease_owner
5403            .clone()
5404            .unwrap_or_else(|| self.worker_id.clone());
5405        let ctx = ActivityContext {
5406            client: self.client.clone(),
5407            task_id: task.task_id,
5408            activity_attempt_id: attempt_id,
5409            lease_owner,
5410            activity_type: task.activity_type,
5411            attempt_number: task.attempt_number,
5412            task_queue: self.task_queue.clone(),
5413            worker_id: self.worker_id.clone(),
5414        };
5415
5416        handler(ctx, args).await
5417    }
5418}
5419
5420fn poller_result(
5421    kind: &str,
5422    result: std::result::Result<Result<()>, tokio::task::JoinError>,
5423) -> Result<()> {
5424    match result {
5425        Ok(result) => result,
5426        Err(error) => Err(Error::WorkerLoop(format!(
5427            "{kind} poller join error: {error}"
5428        ))),
5429    }
5430}
5431
5432fn optional_poller_result(
5433    kind: &str,
5434    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5435) -> Result<()> {
5436    match result {
5437        Some(result) => poller_result(kind, result),
5438        None => Ok(()),
5439    }
5440}
5441
5442async fn join_pollers(
5443    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5444    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5445    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5446) -> Result<()> {
5447    let mut first_error = None;
5448
5449    if let Some(handle) = workflow_poller {
5450        if let Err(error) = poller_result("workflow", handle.await) {
5451            first_error.get_or_insert(error);
5452        }
5453    }
5454
5455    if let Some(handle) = activity_poller {
5456        if let Err(error) = poller_result("activity", handle.await) {
5457            first_error.get_or_insert(error);
5458        }
5459    }
5460
5461    if let Some(handle) = query_poller {
5462        if let Err(error) = poller_result("query", handle.await) {
5463            first_error.get_or_insert(error);
5464        }
5465    }
5466
5467    if let Some(error) = first_error {
5468        Err(error)
5469    } else {
5470        Ok(())
5471    }
5472}
5473
5474fn default_worker_id() -> String {
5475    let millis = SystemTime::now()
5476        .duration_since(UNIX_EPOCH)
5477        .unwrap_or_default()
5478        .as_millis();
5479    format!("rust-worker-{}-{millis}", std::process::id())
5480}
5481
5482fn percent_encode_path_segment(segment: &str) -> String {
5483    const HEX: &[u8; 16] = b"0123456789ABCDEF";
5484    let mut encoded = String::with_capacity(segment.len());
5485
5486    for byte in segment.bytes() {
5487        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5488            encoded.push(char::from(byte));
5489        } else {
5490            encoded.push('%');
5491            encoded.push(char::from(HEX[(byte >> 4) as usize]));
5492            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5493        }
5494    }
5495
5496    encoded
5497}
5498
5499fn unique_request_id(prefix: &str) -> String {
5500    let nanos = SystemTime::now()
5501        .duration_since(UNIX_EPOCH)
5502        .unwrap_or_default()
5503        .as_nanos();
5504    format!("{prefix}-{}-{nanos}", std::process::id())
5505}
5506
5507#[derive(Debug)]
5508struct QueryTaskExecutionFailure {
5509    reason: String,
5510    message: String,
5511    failure_type: String,
5512}
5513
5514impl QueryTaskExecutionFailure {
5515    fn new(
5516        reason: impl Into<String>,
5517        message: impl Into<String>,
5518        failure_type: impl Into<String>,
5519    ) -> Self {
5520        Self {
5521            reason: reason.into(),
5522            message: message.into(),
5523            failure_type: failure_type.into(),
5524        }
5525    }
5526}
5527
5528/// Typed local state owned by one deterministic workflow invocation.
5529///
5530/// Use [`WorkflowInstance::update`] for the same state transitions during
5531/// ordinary execution and replay. A replayed query receives a detached
5532/// immutable `Arc<S>` rather than this mutation-capable handle.
5533#[derive(Clone, Debug)]
5534pub struct WorkflowInstance<S> {
5535    state: Arc<Mutex<S>>,
5536}
5537
5538impl<S> WorkflowInstance<S> {
5539    fn new(state: S) -> Self {
5540        Self {
5541            state: Arc::new(Mutex::new(state)),
5542        }
5543    }
5544
5545    /// Read the current workflow-instance state without changing it.
5546    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5547        let state = self
5548            .state
5549            .lock()
5550            .map_err(|_| Error::WorkflowStatePoisoned)?;
5551        Ok(reader(&state))
5552    }
5553
5554    /// Apply one deterministic workflow-instance state transition.
5555    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5556        let mut state = self
5557            .state
5558            .lock()
5559            .map_err(|_| Error::WorkflowStatePoisoned)?;
5560        Ok(transition(&mut state))
5561    }
5562}
5563
5564impl<S: Clone> WorkflowInstance<S> {
5565    fn snapshot(&self) -> Result<S> {
5566        self.read(Clone::clone)
5567    }
5568}
5569
5570#[derive(Clone, Debug)]
5571pub struct WorkflowContext {
5572    state: Arc<Mutex<WorkflowState>>,
5573}
5574
5575impl WorkflowContext {
5576    /// Identity of the parent workflow currently being replayed.
5577    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5578        let state = self
5579            .state
5580            .lock()
5581            .map_err(|_| Error::WorkflowStatePoisoned)?;
5582        Ok(WorkflowIdentity {
5583            workflow_id: state.workflow_id.clone(),
5584            run_id: state.run_id.clone(),
5585        })
5586    }
5587
5588    /// Return the server-published history budget for this workflow task.
5589    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5590        let state = self
5591            .state
5592            .lock()
5593            .map_err(|_| Error::WorkflowStatePoisoned)?;
5594        Ok(state.history_budget.clone())
5595    }
5596
5597    /// Continue this workflow instance as a fresh run with replacement arguments.
5598    ///
5599    /// Return this value directly from the workflow handler. The worker converts
5600    /// it to the terminal protocol command only after replay has consumed every
5601    /// recorded durable command.
5602    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5603        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5604    }
5605
5606    /// Continue as new with optional workflow-type and task-queue overrides.
5607    pub fn continue_as_new_with_options<T: Serialize>(
5608        &self,
5609        options: ContinueAsNewOptions,
5610        args: T,
5611    ) -> Result<Value> {
5612        options.validate()?;
5613        Err(Error::ContinueAsNew(ContinueAsNewRequest {
5614            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5615            options,
5616        }))
5617    }
5618
5619    pub fn activity<T: Serialize>(
5620        &self,
5621        activity_type: impl Into<String>,
5622        args: T,
5623    ) -> ActivityCall {
5624        self.activity_with_options(activity_type, ActivityOptions::new(), args)
5625    }
5626
5627    pub fn activity_on_queue<T, Q>(
5628        &self,
5629        activity_type: impl Into<String>,
5630        task_queue: Option<Q>,
5631        args: T,
5632    ) -> ActivityCall
5633    where
5634        T: Serialize,
5635        Q: Into<String>,
5636    {
5637        let mut options = ActivityOptions::new();
5638        options.task_queue = task_queue.map(Into::into);
5639        self.activity_with_options(activity_type, options, args)
5640    }
5641
5642    /// Schedule one durable activity with retry, routing, and timeout options.
5643    ///
5644    /// Options are validated before the command is emitted. Once the command is
5645    /// recorded, replay consumes the same activity lifecycle at this command
5646    /// position and never emits a duplicate schedule.
5647    ///
5648    /// ```no_run
5649    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
5650    /// # use std::time::Duration;
5651    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
5652    /// let result = ctx
5653    ///     .activity_with_options(
5654    ///         "charge-card",
5655    ///         ActivityOptions::new()
5656    ///             .task_queue("payments")
5657    ///             .retry_policy(
5658    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
5659    ///                     Duration::from_secs(1),
5660    ///                     2,
5661    ///                     Some(Duration::from_secs(30)),
5662    ///                 ),
5663    ///             )
5664    ///             .start_to_close_timeout(Duration::from_secs(60))
5665    ///             .schedule_to_close_timeout(Duration::from_secs(180))
5666    ///             .heartbeat_timeout(Duration::from_secs(15)),
5667    ///         json!([{"order_id": "order-42"}]),
5668    ///     )
5669    ///     .await;
5670    /// match result {
5671    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
5672    ///         "reason": failure.reason,
5673    ///         "timeout_kind": failure.timeout_kind,
5674    ///     })),
5675    ///     other => other,
5676    /// }
5677    /// # }
5678    /// ```
5679    pub fn activity_with_options<T: Serialize>(
5680        &self,
5681        activity_type: impl Into<String>,
5682        options: ActivityOptions,
5683        args: T,
5684    ) -> ActivityCall {
5685        ActivityCall {
5686            ctx: self.clone(),
5687            activity_type: activity_type.into(),
5688            options,
5689            args: Some(AvroValue::from_serialize(&args)),
5690            scheduled: false,
5691        }
5692    }
5693
5694    pub async fn activity_avro_value<T: Serialize>(
5695        &self,
5696        activity_type: impl Into<String>,
5697        args: T,
5698    ) -> Result<AvroValue> {
5699        let mut call = self.activity(activity_type, args);
5700        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5701    }
5702
5703    pub async fn activity_avro_value_with_options<T: Serialize>(
5704        &self,
5705        activity_type: impl Into<String>,
5706        options: ActivityOptions,
5707        args: T,
5708    ) -> Result<AvroValue> {
5709        let mut call = self.activity_with_options(activity_type, options, args);
5710        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5711    }
5712
5713    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5714        SignalCall {
5715            ctx: self.clone(),
5716            signal_name: signal_name.into(),
5717            opened_wait: false,
5718            matched_pending: false,
5719        }
5720    }
5721
5722    pub async fn wait_signal_avro_value(
5723        &self,
5724        signal_name: impl Into<String>,
5725    ) -> Result<Vec<AvroValue>> {
5726        let mut call = self.wait_signal(signal_name);
5727        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5728    }
5729
5730    /// Wait for server-backed durable time without blocking the worker executor.
5731    ///
5732    /// Polling this future emits one `start_timer` command and yields. The
5733    /// server records the deadline, so neither worker nor server restarts reset
5734    /// the wait. Replay resolves the future only from a `TimerScheduled` and
5735    /// `TimerFired` pair at the same position in the shared durable-command
5736    /// stream, with matching sequence, timer identity, and delay. Sub-second
5737    /// durations round up because protocol deadlines use whole seconds.
5738    ///
5739    /// ```no_run
5740    /// # use durable_workflow::{json, Client, Worker};
5741    /// # use std::time::Duration;
5742    /// # fn configure(client: Client) {
5743    /// let mut worker = Worker::new(client, "rust-workers");
5744    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
5745    ///     ctx.sleep(Duration::from_secs(5)).await?;
5746    ///     Ok(json!({"status": "timer fired"}))
5747    /// });
5748    /// # }
5749    /// ```
5750    pub fn sleep(&self, duration: Duration) -> TimerCall {
5751        let delay_seconds = duration
5752            .as_secs()
5753            .checked_add(u64::from(duration.subsec_nanos() > 0));
5754        TimerCall {
5755            ctx: self.clone(),
5756            delay_seconds,
5757            scheduled: false,
5758            matched_pending: false,
5759        }
5760    }
5761
5762    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
5763    pub fn start_timer(&self, duration: Duration) -> TimerCall {
5764        self.sleep(duration)
5765    }
5766
5767    /// Evaluate a non-deterministic callback once and durably record its typed value.
5768    ///
5769    /// On replay the callback is not invoked: the value is decoded from the
5770    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
5771    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
5772    /// small values that must remain fixed for the lifetime of a workflow run.
5773    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5774    where
5775        T: Serialize + DeserializeOwned,
5776        F: FnOnce() -> T,
5777    {
5778        {
5779            let mut state = self
5780                .state
5781                .lock()
5782                .map_err(|_| Error::WorkflowStatePoisoned)?;
5783            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5784                return match recorded {
5785                    RecordedCommand::SideEffect { sequence, value } => {
5786                        state.command_cursor += 1;
5787                        value.deserialize().map_err(|error| {
5788                            Error::NonDeterministicReplay(ReplayFailure::new(
5789                                "side_effect_type_mismatch",
5790                                Some(sequence),
5791                                Some(std::any::type_name::<T>().to_string()),
5792                                Some(error.to_string()),
5793                                "recorded side-effect value is incompatible with the requested Rust type",
5794                            ))
5795                        })
5796                    }
5797                    other => Err(command_mismatch(&other, "side effect")),
5798                };
5799            }
5800        }
5801
5802        let value = callback();
5803        let avro_value = AvroValue::from_serialize(&value)?;
5804        let mut state = self
5805            .state
5806            .lock()
5807            .map_err(|_| Error::WorkflowStatePoisoned)?;
5808        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5809        state.commands.push(json!({
5810            "type": "record_side_effect",
5811            "result": result,
5812        }));
5813        Ok(value)
5814    }
5815
5816    /// Record or replay a lossless fixed Avro Value side effect.
5817    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5818    where
5819        F: FnOnce() -> AvroValue,
5820    {
5821        {
5822            let mut state = self
5823                .state
5824                .lock()
5825                .map_err(|_| Error::WorkflowStatePoisoned)?;
5826            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5827                return match recorded {
5828                    RecordedCommand::SideEffect { value, .. } => {
5829                        state.command_cursor += 1;
5830                        Ok(value)
5831                    }
5832                    other => Err(command_mismatch(&other, "side effect")),
5833                };
5834            }
5835        }
5836
5837        let value = callback();
5838        let mut state = self
5839            .state
5840            .lock()
5841            .map_err(|_| Error::WorkflowStatePoisoned)?;
5842        let result = encode_typed_envelope(&value, &state.payload_codec)?;
5843        state.commands.push(json!({
5844            "type": "record_side_effect",
5845            "result": result,
5846        }));
5847        Ok(value)
5848    }
5849
5850    /// Record a UUIDv4 once and return the same UUID on every replay.
5851    pub fn uuid_v4(&self) -> Result<Uuid> {
5852        self.side_effect(Uuid::new_v4)
5853    }
5854
5855    /// Select the newest supported version for a change, or replay the version
5856    /// already committed for that stable change ID.
5857    pub fn get_version(
5858        &self,
5859        change_id: impl Into<String>,
5860        min_supported: i32,
5861        max_supported: i32,
5862    ) -> Result<i32> {
5863        let change_id = change_id.into();
5864        if change_id.trim().is_empty() {
5865            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5866                "version_change_id_invalid",
5867                None,
5868                Some("non-empty change ID".to_string()),
5869                Some(change_id),
5870                "version markers require a stable non-empty change ID",
5871            )));
5872        }
5873        if min_supported > max_supported {
5874            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5875                "version_range_invalid",
5876                None,
5877                Some("min_supported <= max_supported".to_string()),
5878                Some(format!("{min_supported}..={max_supported}")),
5879                "version marker supported range is invalid",
5880            )));
5881        }
5882
5883        let mut state = self
5884            .state
5885            .lock()
5886            .map_err(|_| Error::WorkflowStatePoisoned)?;
5887        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5888            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5889            return Ok(version);
5890        }
5891
5892        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5893            return match recorded {
5894                RecordedCommand::VersionMarker {
5895                    sequence,
5896                    change_id: recorded_change_id,
5897                    version,
5898                    ..
5899                } => {
5900                    if recorded_change_id != change_id {
5901                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5902                            "version_change_id_mismatch",
5903                            Some(sequence),
5904                            Some(recorded_change_id),
5905                            Some(change_id),
5906                            "recorded version marker change ID differs from current workflow code",
5907                        )));
5908                    }
5909                    ensure_version_supported(
5910                        &change_id,
5911                        version,
5912                        min_supported,
5913                        max_supported,
5914                        sequence,
5915                    )?;
5916                    state.command_cursor += 1;
5917                    state.version_markers.insert(change_id, (version, sequence));
5918                    Ok(version)
5919                }
5920                other => Err(command_mismatch(
5921                    &other,
5922                    format!("version marker:{change_id}"),
5923                )),
5924            };
5925        }
5926
5927        let version = max_supported;
5928        state.commands.push(json!({
5929            "type": "record_version_marker",
5930            "change_id": change_id,
5931            "version": version,
5932            "min_supported": min_supported,
5933            "max_supported": max_supported,
5934        }));
5935        // Sequence numbers are assigned by the server. Zero identifies a marker
5936        // selected in this uncommitted decision batch for duplicate-call checks.
5937        state.version_markers.insert(change_id, (version, 0));
5938        Ok(version)
5939    }
5940
5941    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
5942    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5943        Ok(self.get_version(change_id, -1, 1)? == 1)
5944    }
5945
5946    /// Keep a patch marker in history after the legacy branch has been removed.
5947    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5948        self.get_version(change_id, -1, 1).map(|_| ())
5949    }
5950
5951    /// Start a named durable child on an explicit queue and await its result.
5952    ///
5953    /// The command is recorded in the parent's sequence-ordered durable command
5954    /// stream. Replay keeps a scheduled child pending without emitting another
5955    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
5956    /// values preserve the history payload codec and include both sides of the
5957    /// durable relationship; failures are returned as
5958    /// [`Error::ChildWorkflowFailed`].
5959    ///
5960    /// ```no_run
5961    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
5962    /// # fn configure(client: Client) {
5963    /// let mut worker = Worker::new(client, "parent-workers");
5964    /// worker.register_workflow("order-parent", |ctx, _input| async move {
5965    ///     let child = ctx
5966    ///         .start_child_workflow(
5967    ///             "fulfil-order",
5968    ///             ChildWorkflowOptions::new("fulfilment-workers")
5969    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
5970    ///             json!([{"order_id": "order-42"}]),
5971    ///         )
5972    ///         .await?;
5973    ///     Ok(child.result)
5974    /// });
5975    /// # }
5976    /// ```
5977    pub fn start_child_workflow<T: Serialize>(
5978        &self,
5979        workflow_type: impl Into<String>,
5980        options: ChildWorkflowOptions,
5981        args: T,
5982    ) -> ChildWorkflowCall {
5983        ChildWorkflowCall {
5984            ctx: self.clone(),
5985            workflow_type: workflow_type.into(),
5986            options,
5987            args: Some(AvroValue::from_serialize(&args)),
5988            scheduled: false,
5989            matched_pending: false,
5990        }
5991    }
5992
5993    pub async fn start_child_workflow_avro_value<T: Serialize>(
5994        &self,
5995        workflow_type: impl Into<String>,
5996        options: ChildWorkflowOptions,
5997        args: T,
5998    ) -> Result<ChildWorkflowAvroResult> {
5999        let mut call = self.start_child_workflow(workflow_type, options, args);
6000        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6001    }
6002
6003    fn take_commands(&self) -> Result<Vec<Value>> {
6004        let mut state = self
6005            .state
6006            .lock()
6007            .map_err(|_| Error::WorkflowStatePoisoned)?;
6008        Ok(std::mem::take(&mut state.commands))
6009    }
6010
6011    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6012        let mut state = self
6013            .state
6014            .lock()
6015            .map_err(|_| Error::WorkflowStatePoisoned)?;
6016
6017        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6018            return Err(command_mismatch(&recorded, "continue as new"));
6019        }
6020        if state.recorded_continue_as_new_sequence.is_some() {
6021            state.continue_as_new_consumed = true;
6022            return Ok(None);
6023        }
6024
6025        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6026        let mut command = serde_json::Map::from_iter([
6027            ("type".to_string(), json!("continue_as_new")),
6028            ("arguments".to_string(), arguments),
6029            ("queue".to_string(), json!(state.task_queue.clone())),
6030        ]);
6031        if let Some(workflow_type) = request.options.workflow_type {
6032            command.insert("workflow_type".to_string(), json!(workflow_type));
6033        }
6034        if let Some(task_queue) = request.options.task_queue {
6035            command.insert("queue".to_string(), json!(task_queue));
6036        }
6037        Ok(Some(Value::Object(command)))
6038    }
6039
6040    fn matched_recorded_pending(&self) -> Result<bool> {
6041        let state = self
6042            .state
6043            .lock()
6044            .map_err(|_| Error::WorkflowStatePoisoned)?;
6045        Ok(state.matched_recorded_pending)
6046    }
6047
6048    fn ensure_history_consumed(&self) -> Result<()> {
6049        let state = self
6050            .state
6051            .lock()
6052            .map_err(|_| Error::WorkflowStatePoisoned)?;
6053        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6054            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6055                "recorded_commands_unconsumed",
6056                Some(command.sequence()),
6057                Some(command.shape().to_string()),
6058                Some("workflow completion".to_string()),
6059                "workflow completed before consuming all recorded durable commands",
6060            )));
6061        }
6062        if let Some(sequence) = state
6063            .recorded_continue_as_new_sequence
6064            .filter(|_| !state.continue_as_new_consumed)
6065        {
6066            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6067                "recorded_continue_as_new_unconsumed",
6068                Some(sequence),
6069                Some("continue as new".to_string()),
6070                Some("workflow completion".to_string()),
6071                "workflow completed without consuming its recorded continue-as-new transition",
6072            )));
6073        }
6074        Ok(())
6075    }
6076}
6077
6078#[derive(Debug)]
6079struct WorkflowState {
6080    workflow_id: Option<String>,
6081    run_id: Option<String>,
6082    task_queue: String,
6083    payload_codec: String,
6084    history_budget: WorkflowHistoryBudget,
6085    resume_signal: Option<ResumeSignal>,
6086    recorded_commands: Vec<RecordedCommand>,
6087    recorded_continue_as_new_sequence: Option<u64>,
6088    continue_as_new_consumed: bool,
6089    command_cursor: usize,
6090    matched_recorded_pending: bool,
6091    version_markers: HashMap<String, (i32, u64)>,
6092    commands: Vec<Value>,
6093}
6094
6095impl WorkflowState {
6096    #[cfg(test)]
6097    fn new(
6098        history: Vec<HistoryEvent>,
6099        task_queue: String,
6100        payload_codec: String,
6101        resume_signal: Option<ResumeSignal>,
6102    ) -> Result<Self> {
6103        Self::new_with_identity(
6104            history,
6105            None,
6106            None,
6107            task_queue,
6108            payload_codec,
6109            resume_signal,
6110        )
6111    }
6112
6113    fn new_with_identity(
6114        history: Vec<HistoryEvent>,
6115        workflow_id: Option<String>,
6116        run_id: Option<String>,
6117        task_queue: String,
6118        payload_codec: String,
6119        resume_signal: Option<ResumeSignal>,
6120    ) -> Result<Self> {
6121        let recorded_commands = recorded_commands(
6122            &history,
6123            &payload_codec,
6124            WorkflowIdentity {
6125                workflow_id: workflow_id.clone(),
6126                run_id: run_id.clone(),
6127            },
6128        )?;
6129        let recorded_continue_as_new = history
6130            .iter()
6131            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6132            .collect::<Vec<_>>();
6133        if recorded_continue_as_new.len() > 1 {
6134            return Err(invalid_recorded_history(
6135                "duplicate_continue_as_new_transition",
6136                recorded_continue_as_new
6137                    .last()
6138                    .and_then(|event| durable_event_sequence(event))
6139                    .unwrap_or(0),
6140                "one WorkflowContinuedAsNew event",
6141                &format!(
6142                    "{} WorkflowContinuedAsNew events",
6143                    recorded_continue_as_new.len()
6144                ),
6145                "workflow history records one continue-as-new transition more than once",
6146            ));
6147        }
6148        let recorded_continue_as_new_sequence = recorded_continue_as_new
6149            .first()
6150            .map(|event| {
6151                durable_event_sequence(event).ok_or_else(|| {
6152                    Error::NonDeterministicReplay(ReplayFailure::new(
6153                        "continue_as_new_sequence_missing",
6154                        None,
6155                        Some("recorded transition sequence".to_string()),
6156                        Some("missing sequence".to_string()),
6157                        "WorkflowContinuedAsNew history is missing its recorded sequence",
6158                    ))
6159                })
6160            })
6161            .transpose()?;
6162        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6163        Ok(Self {
6164            workflow_id,
6165            run_id,
6166            task_queue,
6167            payload_codec,
6168            history_budget: WorkflowHistoryBudget {
6169                event_count,
6170                ..WorkflowHistoryBudget::default()
6171            },
6172            resume_signal,
6173            recorded_commands,
6174            recorded_continue_as_new_sequence,
6175            continue_as_new_consumed: false,
6176            command_cursor: 0,
6177            matched_recorded_pending: false,
6178            version_markers: HashMap::new(),
6179            commands: Vec::new(),
6180        })
6181    }
6182}
6183
6184#[derive(Clone, Debug)]
6185enum RecordedCommand {
6186    Activity {
6187        sequence: u64,
6188        activity_type: Option<String>,
6189        options: Option<RecordedActivityOptions>,
6190        outcome: Option<ActivityOutcome>,
6191    },
6192    Timer {
6193        sequence: u64,
6194        delay_seconds: u64,
6195        fired: bool,
6196    },
6197    ChildWorkflow {
6198        sequence: u64,
6199        workflow_type: Option<String>,
6200        outcome: Option<ChildWorkflowOutcome>,
6201    },
6202    SignalWait {
6203        sequence: u64,
6204        signal_name: String,
6205        value: Option<Vec<AvroValue>>,
6206    },
6207    SideEffect {
6208        sequence: u64,
6209        value: AvroValue,
6210    },
6211    VersionMarker {
6212        sequence: u64,
6213        change_id: String,
6214        version: i32,
6215    },
6216}
6217
6218#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6219struct RecordedActivityOptions {
6220    task_queue: RecordedSnapshotValue<Option<String>>,
6221    execution_mode: RecordedSnapshotValue<Option<String>>,
6222    retry_policy: ActivityRetrySnapshot,
6223}
6224
6225#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6226enum RecordedSnapshotValue<T> {
6227    /// Older history did not persist this field, so it cannot constrain replay.
6228    Unknown,
6229    Known(T),
6230}
6231
6232impl<T: PartialEq> RecordedSnapshotValue<T> {
6233    fn matches_current(&self, current: &Self) -> bool {
6234        match self {
6235            Self::Unknown => true,
6236            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6237        }
6238    }
6239}
6240
6241#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6242struct ActivityRetrySnapshot {
6243    snapshot_version: RecordedSnapshotValue<Option<u64>>,
6244    max_attempts: RecordedSnapshotValue<Option<u64>>,
6245    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6246    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6247    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6248    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6249    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6250    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6251}
6252
6253impl ActivityRetrySnapshot {
6254    fn matches_current(&self, current: &Self) -> bool {
6255        self.snapshot_version
6256            .matches_current(&current.snapshot_version)
6257            && self.max_attempts.matches_current(&current.max_attempts)
6258            && self
6259                .backoff_seconds
6260                .matches_current(&current.backoff_seconds)
6261            && self
6262                .start_to_close_timeout
6263                .matches_current(&current.start_to_close_timeout)
6264            && self
6265                .schedule_to_start_timeout
6266                .matches_current(&current.schedule_to_start_timeout)
6267            && self
6268                .schedule_to_close_timeout
6269                .matches_current(&current.schedule_to_close_timeout)
6270            && self
6271                .heartbeat_timeout
6272                .matches_current(&current.heartbeat_timeout)
6273            && self
6274                .non_retryable_error_types
6275                .matches_current(&current.non_retryable_error_types)
6276    }
6277}
6278
6279fn recorded_optional_u64(
6280    object: Option<&serde_json::Map<String, Value>>,
6281    field: &str,
6282) -> RecordedSnapshotValue<Option<u64>> {
6283    match object.and_then(|object| object.get(field)) {
6284        None => RecordedSnapshotValue::Unknown,
6285        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6286        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6287    }
6288}
6289
6290fn recorded_optional_string(
6291    object: &serde_json::Map<String, Value>,
6292    field: &str,
6293) -> RecordedSnapshotValue<Option<String>> {
6294    match object.get(field) {
6295        None => RecordedSnapshotValue::Unknown,
6296        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6297        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6298    }
6299}
6300
6301fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6302    let policy = policy.and_then(Value::as_object);
6303    let backoff_seconds = policy
6304        .and_then(|policy| policy.get("backoff_seconds"))
6305        .and_then(Value::as_array)
6306        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6307        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6308    let mut non_retryable_error_types = Vec::new();
6309    for error_type in policy
6310        .and_then(|policy| policy.get("non_retryable_error_types"))
6311        .and_then(Value::as_array)
6312        .into_iter()
6313        .flatten()
6314        .filter_map(Value::as_str)
6315        .map(str::trim)
6316        .filter(|error_type| !error_type.is_empty())
6317    {
6318        if !non_retryable_error_types
6319            .iter()
6320            .any(|recorded| recorded == error_type)
6321        {
6322            non_retryable_error_types.push(error_type.to_string());
6323        }
6324    }
6325
6326    ActivityRetrySnapshot {
6327        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6328        max_attempts: recorded_optional_u64(policy, "max_attempts"),
6329        backoff_seconds,
6330        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6331        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6332        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6333        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6334        non_retryable_error_types: if policy
6335            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6336        {
6337            RecordedSnapshotValue::Known(non_retryable_error_types)
6338        } else {
6339            RecordedSnapshotValue::Unknown
6340        },
6341    }
6342}
6343
6344fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6345    let policy = options.retry_policy.as_ref();
6346    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6347        Some(Value::Null) => None,
6348        Some(value) => value_as_u64(value),
6349        None => Some(1),
6350    };
6351    let backoff_seconds = policy
6352        .and_then(|policy| policy.get("backoff_seconds"))
6353        .and_then(Value::as_array)
6354        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6355        .unwrap_or_default();
6356    let non_retryable_error_types = policy
6357        .and_then(|policy| policy.get("non_retryable_error_types"))
6358        .and_then(Value::as_array)
6359        .into_iter()
6360        .flatten()
6361        .filter_map(Value::as_str)
6362        .map(str::to_string)
6363        .collect();
6364
6365    ActivityRetrySnapshot {
6366        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6367        max_attempts: RecordedSnapshotValue::Known(max_attempts),
6368        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6369        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6370        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6371        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6372        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6373        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6374    }
6375}
6376
6377fn activity_options_description(options: &RecordedActivityOptions) -> String {
6378    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6379}
6380
6381impl RecordedCommand {
6382    fn sequence(&self) -> u64 {
6383        match self {
6384            Self::Activity { sequence, .. }
6385            | Self::Timer { sequence, .. }
6386            | Self::ChildWorkflow { sequence, .. }
6387            | Self::SignalWait { sequence, .. }
6388            | Self::SideEffect { sequence, .. }
6389            | Self::VersionMarker { sequence, .. } => *sequence,
6390        }
6391    }
6392
6393    fn shape(&self) -> &'static str {
6394        match self {
6395            Self::Activity { .. } => "activity",
6396            Self::Timer { .. } => "timer",
6397            Self::ChildWorkflow { .. } => "child workflow",
6398            Self::SignalWait { .. } => "signal wait",
6399            Self::SideEffect { .. } => "side effect",
6400            Self::VersionMarker { .. } => "version marker",
6401        }
6402    }
6403}
6404
6405fn ensure_version_supported(
6406    change_id: &str,
6407    version: i32,
6408    min_supported: i32,
6409    max_supported: i32,
6410    sequence: u64,
6411) -> Result<()> {
6412    if (min_supported..=max_supported).contains(&version) {
6413        return Ok(());
6414    }
6415    Err(Error::NonDeterministicReplay(ReplayFailure::new(
6416        "version_marker_incompatible_range",
6417        (sequence != 0).then_some(sequence),
6418        Some(format!("{min_supported}..={max_supported}")),
6419        Some(format!("{change_id}:{version}")),
6420        "recorded workflow version is outside the range supported by current code",
6421    )))
6422}
6423
6424#[derive(Clone, Debug)]
6425struct ResumeSignal {
6426    signal_name: String,
6427    arguments: Vec<AvroValue>,
6428}
6429
6430pub struct ActivityCall {
6431    ctx: WorkflowContext,
6432    activity_type: String,
6433    options: ActivityOptions,
6434    args: Option<Result<AvroValue>>,
6435    scheduled: bool,
6436}
6437
6438impl ActivityCall {
6439    fn poll_avro_value(
6440        mut self: Pin<&mut Self>,
6441        _cx: &mut TaskContext<'_>,
6442    ) -> Poll<Result<AvroValue>> {
6443        let ctx = self.ctx.clone();
6444        let mut state = match ctx.state.lock() {
6445            Ok(state) => state,
6446            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6447        };
6448
6449        if self.scheduled {
6450            return Poll::Pending;
6451        }
6452
6453        let options = match self.options.validate() {
6454            Ok(options) => options,
6455            Err(error) => {
6456                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6457            }
6458        };
6459        let task_queue = options
6460            .task_queue
6461            .clone()
6462            .unwrap_or_else(|| state.task_queue.clone());
6463        let current_recorded_options = RecordedActivityOptions {
6464            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6465            // Rust schedules ordinary durable activities. The server records a
6466            // non-null mode only for a specialized execution primitive.
6467            execution_mode: RecordedSnapshotValue::Known(None),
6468            retry_policy: current_activity_retry_snapshot(&options),
6469        };
6470
6471        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6472            let sequence = recorded.sequence();
6473            match recorded {
6474                RecordedCommand::Activity {
6475                    activity_type,
6476                    options: recorded_options,
6477                    outcome,
6478                    ..
6479                } => {
6480                    if let Some(recorded_type) = activity_type {
6481                        if recorded_type != self.activity_type {
6482                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6483                                ReplayFailure::new(
6484                                    "recorded_command_detail_mismatch",
6485                                    Some(sequence),
6486                                    Some(format!("activity:{recorded_type}")),
6487                                    Some(format!("activity:{}", self.activity_type)),
6488                                    "recorded activity type differs from the current workflow command",
6489                                ),
6490                            )));
6491                        }
6492                    }
6493                    if let Some(recorded_options) = recorded_options {
6494                        if !recorded_options
6495                            .task_queue
6496                            .matches_current(&current_recorded_options.task_queue)
6497                        {
6498                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6499                                ReplayFailure::new(
6500                                    "activity_task_queue_mismatch",
6501                                    Some(sequence),
6502                                    Some(activity_options_description(&recorded_options)),
6503                                    Some(activity_options_description(&current_recorded_options)),
6504                                    "recorded activity task queue differs from the current workflow command",
6505                                ),
6506                            )));
6507                        }
6508                        if !recorded_options
6509                            .execution_mode
6510                            .matches_current(&current_recorded_options.execution_mode)
6511                        {
6512                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6513                                ReplayFailure::new(
6514                                    "activity_execution_mode_mismatch",
6515                                    Some(sequence),
6516                                    Some(activity_options_description(&recorded_options)),
6517                                    Some(activity_options_description(&current_recorded_options)),
6518                                    "recorded activity execution mode differs from the current workflow command",
6519                                ),
6520                            )));
6521                        }
6522                        if !recorded_options
6523                            .retry_policy
6524                            .matches_current(&current_recorded_options.retry_policy)
6525                        {
6526                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6527                                ReplayFailure::new(
6528                                    "activity_retry_policy_mismatch",
6529                                    Some(sequence),
6530                                    Some(activity_options_description(&recorded_options)),
6531                                    Some(activity_options_description(&current_recorded_options)),
6532                                    "recorded activity retry policy differs from the current workflow command",
6533                                ),
6534                            )));
6535                        }
6536                    }
6537                    state.command_cursor += 1;
6538                    if let Some(outcome) = outcome {
6539                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6540                    }
6541                    state.matched_recorded_pending = true;
6542                    self.scheduled = true;
6543                    return Poll::Pending;
6544                }
6545                other => {
6546                    return Poll::Ready(Err(command_mismatch(
6547                        &other,
6548                        format!("activity:{}", self.activity_type),
6549                    )));
6550                }
6551            }
6552        }
6553
6554        if !self.scheduled {
6555            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6556                Ok(args) => args,
6557                Err(error) => return Poll::Ready(Err(error)),
6558            };
6559            let arguments = normalize_avro_arguments(args);
6560            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6561                Ok(envelope) => envelope,
6562                Err(error) => return Poll::Ready(Err(error)),
6563            };
6564
6565            let mut command = serde_json::Map::from_iter([
6566                ("type".to_string(), json!("schedule_activity")),
6567                (
6568                    "activity_type".to_string(),
6569                    json!(self.activity_type.clone()),
6570                ),
6571                ("queue".to_string(), json!(task_queue)),
6572                ("arguments".to_string(), envelope),
6573            ]);
6574            for (field, value) in [
6575                ("start_to_close_timeout", options.start_to_close_timeout),
6576                (
6577                    "schedule_to_start_timeout",
6578                    options.schedule_to_start_timeout,
6579                ),
6580                (
6581                    "schedule_to_close_timeout",
6582                    options.schedule_to_close_timeout,
6583                ),
6584                ("heartbeat_timeout", options.heartbeat_timeout),
6585            ] {
6586                if let Some(value) = value {
6587                    command.insert(field.to_string(), json!(value));
6588                }
6589            }
6590            if let Some(retry_policy) = options.retry_policy {
6591                command.insert("retry_policy".to_string(), retry_policy);
6592            }
6593            state.commands.push(Value::Object(command));
6594            self.scheduled = true;
6595        }
6596
6597        Poll::Pending
6598    }
6599}
6600
6601impl Future for ActivityCall {
6602    type Output = Result<Value>;
6603
6604    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6605        match self.poll_avro_value(cx) {
6606            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6607            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6608            Poll::Pending => Poll::Pending,
6609        }
6610    }
6611}
6612
6613/// Future returned by [`WorkflowContext::sleep`].
6614pub struct TimerCall {
6615    ctx: WorkflowContext,
6616    delay_seconds: Option<u64>,
6617    scheduled: bool,
6618    matched_pending: bool,
6619}
6620
6621impl Future for TimerCall {
6622    type Output = Result<()>;
6623
6624    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6625        if self.matched_pending {
6626            return Poll::Pending;
6627        }
6628
6629        let ctx = self.ctx.clone();
6630        let Some(requested_delay) = self.delay_seconds else {
6631            return Poll::Ready(Err(Error::TimerDurationOverflow));
6632        };
6633        let mut state = match ctx.state.lock() {
6634            Ok(state) => state,
6635            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6636        };
6637
6638        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6639            match recorded {
6640                RecordedCommand::Timer {
6641                    sequence,
6642                    delay_seconds,
6643                    fired,
6644                    ..
6645                } => {
6646                    if delay_seconds != requested_delay {
6647                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6648                            ReplayFailure::new(
6649                                "timer_delay_mismatch",
6650                                Some(sequence),
6651                                Some(format!("timer:{delay_seconds}s")),
6652                                Some(format!("timer:{requested_delay}s")),
6653                                "recorded timer delay differs from the current workflow command",
6654                            ),
6655                        )));
6656                    }
6657                    state.command_cursor += 1;
6658                    if fired {
6659                        return Poll::Ready(Ok(()));
6660                    }
6661                    state.matched_recorded_pending = true;
6662                    self.scheduled = true;
6663                    self.matched_pending = true;
6664                    return Poll::Pending;
6665                }
6666                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6667            }
6668        }
6669
6670        if !self.scheduled {
6671            state.commands.push(json!({
6672                "type": "start_timer",
6673                "delay_seconds": requested_delay,
6674            }));
6675            self.scheduled = true;
6676        }
6677
6678        Poll::Pending
6679    }
6680}
6681
6682/// Future returned by [`WorkflowContext::start_child_workflow`].
6683pub struct ChildWorkflowCall {
6684    ctx: WorkflowContext,
6685    workflow_type: String,
6686    options: ChildWorkflowOptions,
6687    args: Option<Result<AvroValue>>,
6688    scheduled: bool,
6689    matched_pending: bool,
6690}
6691
6692impl ChildWorkflowCall {
6693    fn poll_avro_value(
6694        mut self: Pin<&mut Self>,
6695        _cx: &mut TaskContext<'_>,
6696    ) -> Poll<Result<ChildWorkflowAvroResult>> {
6697        if self.matched_pending {
6698            return Poll::Pending;
6699        }
6700
6701        let ctx = self.ctx.clone();
6702        let mut state = match ctx.state.lock() {
6703            Ok(state) => state,
6704            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6705        };
6706
6707        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6708            let sequence = recorded.sequence();
6709            match recorded {
6710                RecordedCommand::ChildWorkflow {
6711                    workflow_type,
6712                    outcome,
6713                    ..
6714                } => {
6715                    if let Some(recorded_type) = workflow_type {
6716                        if recorded_type != self.workflow_type {
6717                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6718                                ReplayFailure::new(
6719                                    "recorded_command_detail_mismatch",
6720                                    Some(sequence),
6721                                    Some(format!("child workflow:{recorded_type}")),
6722                                    Some(format!("child workflow:{}", self.workflow_type)),
6723                                    "recorded child workflow type differs from the current workflow command",
6724                                ),
6725                            )));
6726                        }
6727                    }
6728                    state.command_cursor += 1;
6729                    if let Some(outcome) = outcome {
6730                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6731                    }
6732                    state.matched_recorded_pending = true;
6733                    self.scheduled = true;
6734                    self.matched_pending = true;
6735                    return Poll::Pending;
6736                }
6737                other => {
6738                    return Poll::Ready(Err(command_mismatch(
6739                        &other,
6740                        format!("child workflow:{}", self.workflow_type),
6741                    )));
6742                }
6743            }
6744        }
6745
6746        if !self.scheduled {
6747            if self.options.task_queue.trim().is_empty() {
6748                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6749                    "task_queue must not be empty".to_string(),
6750                )));
6751            }
6752            for (name, value) in [
6753                (
6754                    "execution_timeout_seconds",
6755                    self.options.execution_timeout_seconds,
6756                ),
6757                ("run_timeout_seconds", self.options.run_timeout_seconds),
6758            ] {
6759                if value == Some(0) {
6760                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6761                        "{name} must be at least 1"
6762                    ))));
6763                }
6764            }
6765
6766            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6767                Ok(args) => args,
6768                Err(error) => return Poll::Ready(Err(error)),
6769            };
6770            let arguments = match encode_typed_envelope(
6771                &normalize_avro_arguments(args),
6772                &state.payload_codec,
6773            ) {
6774                Ok(arguments) => arguments,
6775                Err(error) => return Poll::Ready(Err(error)),
6776            };
6777            let mut command = json!({
6778                "type": "start_child_workflow",
6779                "workflow_type": self.workflow_type,
6780                "queue": self.options.task_queue,
6781                "parent_close_policy": self.options.parent_close_policy.as_str(),
6782                "arguments": arguments,
6783            });
6784            let object = command
6785                .as_object_mut()
6786                .expect("child workflow command is always an object");
6787            if let Some(policy) = &self.options.retry_policy {
6788                let mut retry_policy = serde_json::Map::new();
6789                if let Some(max_attempts) = policy.max_attempts {
6790                    if max_attempts == 0 {
6791                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6792                            "retry_policy.max_attempts must be at least 1".to_string(),
6793                        )));
6794                    }
6795                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6796                }
6797                if !policy.backoff_seconds.is_empty() {
6798                    retry_policy
6799                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6800                }
6801                if !policy.non_retryable_error_types.is_empty() {
6802                    retry_policy.insert(
6803                        "non_retryable_error_types".to_string(),
6804                        json!(policy.non_retryable_error_types),
6805                    );
6806                }
6807                if retry_policy.is_empty() {
6808                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6809                        "retry_policy must configure at least one field".to_string(),
6810                    )));
6811                }
6812                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6813            }
6814            if let Some(seconds) = self.options.execution_timeout_seconds {
6815                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6816            }
6817            if let Some(seconds) = self.options.run_timeout_seconds {
6818                object.insert("run_timeout_seconds".to_string(), json!(seconds));
6819            }
6820            state.commands.push(command);
6821            self.scheduled = true;
6822        }
6823
6824        Poll::Pending
6825    }
6826}
6827
6828impl Future for ChildWorkflowCall {
6829    type Output = Result<ChildWorkflowResult>;
6830
6831    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6832        match self.poll_avro_value(cx) {
6833            Poll::Ready(Ok(result)) => match result.result.into_json() {
6834                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6835                    parent: result.parent,
6836                    child: result.child,
6837                    child_workflow_type: result.child_workflow_type,
6838                    result: projected,
6839                })),
6840                Err(error) => Poll::Ready(Err(error)),
6841            },
6842            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6843            Poll::Pending => Poll::Pending,
6844        }
6845    }
6846}
6847
6848fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6849    Error::NonDeterministicReplay(ReplayFailure::new(
6850        "recorded_command_mismatch",
6851        Some(recorded.sequence()),
6852        Some(recorded.shape().to_string()),
6853        Some(actual.into()),
6854        "current workflow command does not match the recorded durable command sequence",
6855    ))
6856}
6857
6858pub struct SignalCall {
6859    ctx: WorkflowContext,
6860    signal_name: String,
6861    opened_wait: bool,
6862    matched_pending: bool,
6863}
6864
6865impl SignalCall {
6866    fn poll_avro_value(
6867        mut self: Pin<&mut Self>,
6868        _cx: &mut TaskContext<'_>,
6869    ) -> Poll<Result<Vec<AvroValue>>> {
6870        if self.matched_pending {
6871            return Poll::Pending;
6872        }
6873
6874        let ctx = self.ctx.clone();
6875        let mut state = match ctx.state.lock() {
6876            Ok(state) => state,
6877            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6878        };
6879
6880        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6881            match recorded {
6882                RecordedCommand::SignalWait {
6883                    sequence,
6884                    signal_name,
6885                    value,
6886                } => {
6887                    if signal_name != self.signal_name {
6888                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6889                            ReplayFailure::new(
6890                                "recorded_command_detail_mismatch",
6891                                Some(sequence),
6892                                Some(format!("signal wait:{signal_name}")),
6893                                Some(format!("signal wait:{}", self.signal_name)),
6894                                "recorded signal name differs from the current workflow command",
6895                            ),
6896                        )));
6897                    }
6898
6899                    state.command_cursor += 1;
6900                    if let Some(value) = value {
6901                        return Poll::Ready(Ok(value));
6902                    }
6903                    if state
6904                        .resume_signal
6905                        .as_ref()
6906                        .is_some_and(|signal| signal.signal_name == self.signal_name)
6907                    {
6908                        let signal = state
6909                            .resume_signal
6910                            .take()
6911                            .expect("matching resume signal is present");
6912                        return Poll::Ready(Ok(signal.arguments));
6913                    }
6914
6915                    state.matched_recorded_pending = true;
6916                    self.opened_wait = true;
6917                    self.matched_pending = true;
6918                    return Poll::Pending;
6919                }
6920                other => {
6921                    return Poll::Ready(Err(command_mismatch(
6922                        &other,
6923                        format!("signal wait:{}", self.signal_name),
6924                    )));
6925                }
6926            }
6927        }
6928
6929        if state
6930            .resume_signal
6931            .as_ref()
6932            .is_some_and(|signal| signal.signal_name == self.signal_name)
6933        {
6934            let signal = state
6935                .resume_signal
6936                .take()
6937                .expect("matching resume signal is present");
6938            return Poll::Ready(Ok(signal.arguments));
6939        }
6940
6941        if !self.opened_wait {
6942            state.commands.push(json!({
6943                "type": "open_signal_wait",
6944                "signal_name": self.signal_name
6945            }));
6946            self.opened_wait = true;
6947        }
6948
6949        Poll::Pending
6950    }
6951}
6952
6953impl Future for SignalCall {
6954    type Output = Result<Vec<Value>>;
6955
6956    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6957        match self.poll_avro_value(cx) {
6958            Poll::Ready(Ok(values)) => Poll::Ready(
6959                values
6960                    .into_iter()
6961                    .map(AvroValue::into_json)
6962                    .collect::<Result<Vec<_>>>(),
6963            ),
6964            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6965            Poll::Pending => Poll::Pending,
6966        }
6967    }
6968}
6969
6970#[derive(Clone, Debug)]
6971pub struct ActivityContext {
6972    client: Client,
6973    pub task_id: String,
6974    pub activity_attempt_id: String,
6975    pub lease_owner: String,
6976    pub activity_type: String,
6977    pub attempt_number: u64,
6978    pub task_queue: String,
6979    pub worker_id: String,
6980}
6981
6982impl ActivityContext {
6983    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
6984        self.client
6985            .heartbeat_activity_task(
6986                &self.task_id,
6987                &self.activity_attempt_id,
6988                &self.lease_owner,
6989                details,
6990            )
6991            .await
6992    }
6993}
6994
6995fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
6996    match value {
6997        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
6998            value, codec,
6999        )?)),
7000        None => Ok(AvroValue::Array(Vec::new())),
7001    }
7002}
7003
7004fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7005    let Some(signal_name) = task
7006        .signal_name
7007        .as_deref()
7008        .filter(|value| !value.is_empty())
7009    else {
7010        return Ok(None);
7011    };
7012    let Some(arguments) = task.signal_arguments.as_ref() else {
7013        return Ok(None);
7014    };
7015
7016    let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
7017    let AvroValue::Array(arguments) = decoded else {
7018        unreachable!("normalize_avro_arguments always returns an array");
7019    };
7020
7021    Ok(Some(ResumeSignal {
7022        signal_name: signal_name.to_string(),
7023        arguments,
7024    }))
7025}
7026
7027fn recorded_commands(
7028    events: &[HistoryEvent],
7029    fallback_codec: &str,
7030    parent: WorkflowIdentity,
7031) -> Result<Vec<RecordedCommand>> {
7032    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7033    let mut last_new_sequence = None;
7034
7035    for event in events {
7036        let is_activity = matches!(
7037            event.event_type.as_str(),
7038            "ActivityScheduled"
7039                | "ActivityStarted"
7040                | "ActivityHeartbeatRecorded"
7041                | "ActivityRetryScheduled"
7042                | "ActivityCompleted"
7043                | "ActivityFailed"
7044                | "ActivityCancelled"
7045                | "ActivityTimedOut"
7046        );
7047        let is_workflow_timer = matches!(
7048            event.event_type.as_str(),
7049            "TimerScheduled" | "TimerCancelled" | "TimerFired"
7050        ) && !is_internal_timer_event(event);
7051        let is_child_workflow = matches!(
7052            event.event_type.as_str(),
7053            "ChildWorkflowScheduled"
7054                | "ChildRunCompleted"
7055                | "ChildRunFailed"
7056                | "ChildRunCancelled"
7057                | "ChildRunTerminated"
7058        );
7059        let is_signal_wait = is_recorded_signal_wait_event(event);
7060        let is_side_effect = event.event_type == "SideEffectRecorded";
7061        let is_version_marker = event.event_type == "VersionMarkerRecorded";
7062        if !is_activity
7063            && !is_workflow_timer
7064            && !is_child_workflow
7065            && !is_signal_wait
7066            && !is_side_effect
7067            && !is_version_marker
7068        {
7069            continue;
7070        }
7071
7072        let sequence = durable_event_sequence(event).ok_or_else(|| {
7073            Error::NonDeterministicReplay(ReplayFailure::new(
7074                "durable_command_sequence_missing",
7075                None,
7076                Some("positive workflow sequence".to_string()),
7077                Some(event.event_type.clone()),
7078                "durable command history event has no workflow sequence",
7079            ))
7080        })?;
7081        if sequence == 0 {
7082            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7083                "durable_command_sequence_invalid",
7084                Some(sequence),
7085                Some("positive workflow sequence".to_string()),
7086                Some(sequence.to_string()),
7087                "durable command history uses an invalid workflow sequence",
7088            )));
7089        }
7090        if !events_by_sequence.contains_key(&sequence) {
7091            if let Some(previous) = last_new_sequence {
7092                if sequence < previous {
7093                    return Err(invalid_recorded_history(
7094                        "durable_command_sequence_mismatch",
7095                        sequence,
7096                        &format!("workflow sequence greater than {previous}"),
7097                        &sequence.to_string(),
7098                        "durable commands are not strictly ordered by their recorded workflow sequence",
7099                    ));
7100                }
7101            }
7102            last_new_sequence = Some(sequence);
7103        }
7104        events_by_sequence.entry(sequence).or_default().push(event);
7105    }
7106
7107    let commands: Vec<RecordedCommand> = events_by_sequence
7108        .into_iter()
7109        .map(|(sequence, sequence_events)| {
7110            let activity_events: Vec<_> = sequence_events
7111                .iter()
7112                .copied()
7113                .filter(|event| event.event_type.starts_with("Activity"))
7114                .collect();
7115            let timer_events: Vec<_> = sequence_events
7116                .iter()
7117                .copied()
7118                .filter(|event| event.event_type.starts_with("Timer"))
7119                .collect();
7120            let child_events: Vec<_> = sequence_events
7121                .iter()
7122                .copied()
7123                .filter(|event| {
7124                    event.event_type == "ChildWorkflowScheduled"
7125                        || event.event_type.starts_with("ChildRun")
7126                })
7127                .collect();
7128            let signal_wait_events: Vec<_> = sequence_events
7129                .iter()
7130                .copied()
7131                .filter(|event| is_recorded_signal_wait_event(event))
7132                .collect();
7133            let side_effect_events: Vec<_> = sequence_events
7134                .iter()
7135                .copied()
7136                .filter(|event| event.event_type == "SideEffectRecorded")
7137                .collect();
7138            let version_marker_events: Vec<_> = sequence_events
7139                .iter()
7140                .copied()
7141                .filter(|event| event.event_type == "VersionMarkerRecorded")
7142                .collect();
7143
7144            let command_kind_count = usize::from(!activity_events.is_empty())
7145                + usize::from(!timer_events.is_empty())
7146                + usize::from(!child_events.is_empty())
7147                + usize::from(!signal_wait_events.is_empty())
7148                + usize::from(!side_effect_events.is_empty())
7149                + usize::from(!version_marker_events.is_empty());
7150            if command_kind_count > 1 {
7151                let actual = [
7152                    (!activity_events.is_empty()).then_some("activity"),
7153                    (!timer_events.is_empty()).then_some("timer"),
7154                    (!child_events.is_empty()).then_some("child workflow"),
7155                    (!signal_wait_events.is_empty()).then_some("signal wait"),
7156                    (!side_effect_events.is_empty()).then_some("side effect"),
7157                    (!version_marker_events.is_empty()).then_some("version marker"),
7158                ]
7159                .into_iter()
7160                .flatten()
7161                .collect::<Vec<_>>()
7162                .join(" and ");
7163                return Err(invalid_recorded_history(
7164                    "durable_command_sequence_collision",
7165                    sequence,
7166                    "one durable command kind",
7167                    &actual,
7168                    "one workflow sequence records more than one durable command kind",
7169                ));
7170            }
7171
7172            if !activity_events.is_empty() {
7173                let scheduled_count = activity_events
7174                    .iter()
7175                    .filter(|event| event.event_type == "ActivityScheduled")
7176                    .count();
7177                if scheduled_count > 1 {
7178                    return Err(invalid_recorded_history(
7179                        "duplicate_activity_schedule",
7180                        sequence,
7181                        "at most one ActivityScheduled event",
7182                        "multiple ActivityScheduled events",
7183                        "activity history schedules more than one command at one workflow sequence",
7184                    ));
7185                }
7186                let activity_type = activity_events.iter().find_map(|event| {
7187                    event
7188                        .payload
7189                        .get("activity_type")
7190                        .or_else(|| event.payload.get("activity_name"))
7191                        .and_then(Value::as_str)
7192                        .map(str::to_string)
7193                });
7194                if activity_events.iter().filter_map(|event| {
7195                    event
7196                        .payload
7197                        .get("activity_type")
7198                        .or_else(|| event.payload.get("activity_name"))
7199                        .and_then(Value::as_str)
7200                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7201                    return Err(invalid_recorded_history(
7202                        "activity_identity_mismatch",
7203                        sequence,
7204                        activity_type.as_deref().unwrap_or("one activity identity"),
7205                        "conflicting activity identities",
7206                        "activity lifecycle events at one workflow sequence disagree on identity",
7207                    ));
7208                }
7209                let terminal: Vec<_> = activity_events
7210                    .iter()
7211                    .copied()
7212                    .filter(|event| {
7213                        matches!(
7214                            event.event_type.as_str(),
7215                            "ActivityCompleted"
7216                                | "ActivityFailed"
7217                                | "ActivityCancelled"
7218                                | "ActivityTimedOut"
7219                        )
7220                    })
7221                    .collect();
7222                if terminal.len() > 1 {
7223                    return Err(invalid_recorded_history(
7224                        "duplicate_activity_terminal_event",
7225                        sequence,
7226                        "at most one terminal activity event",
7227                        "multiple terminal activity events",
7228                        "activity history settles one command more than once",
7229                    ));
7230                }
7231                let outcome = terminal
7232                    .first()
7233                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7234                    .transpose()?;
7235                let options = activity_events
7236                    .iter()
7237                    .find(|event| event.event_type == "ActivityScheduled")
7238                    .and_then(|event| event.payload.get("activity"))
7239                    .and_then(Value::as_object)
7240                    .map(|activity| RecordedActivityOptions {
7241                        task_queue: recorded_optional_string(activity, "queue"),
7242                        execution_mode: recorded_optional_string(activity, "execution_mode"),
7243                        retry_policy: recorded_activity_retry_snapshot(
7244                            activity.get("retry_policy"),
7245                        ),
7246                    });
7247                return Ok(RecordedCommand::Activity {
7248                    sequence,
7249                    activity_type,
7250                    options,
7251                    outcome,
7252                });
7253            }
7254
7255            if !child_events.is_empty() {
7256                let scheduled: Vec<_> = child_events
7257                    .iter()
7258                    .copied()
7259                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
7260                    .collect();
7261                if scheduled.len() != 1 {
7262                    return Err(invalid_recorded_history(
7263                        "child_workflow_schedule_missing_or_duplicate",
7264                        sequence,
7265                        "one ChildWorkflowScheduled event",
7266                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7267                        "child workflow replay requires exactly one recorded schedule event",
7268                    ));
7269                }
7270                let workflow_type = child_events.iter().find_map(|event| {
7271                    event
7272                        .payload
7273                        .get("child_workflow_type")
7274                        .or_else(|| event.payload.get("workflow_type"))
7275                        .and_then(Value::as_str)
7276                        .filter(|value| !value.is_empty())
7277                        .map(str::to_string)
7278                });
7279                if child_events
7280                    .iter()
7281                    .filter_map(|event| {
7282                        event
7283                            .payload
7284                            .get("child_workflow_type")
7285                            .or_else(|| event.payload.get("workflow_type"))
7286                            .and_then(Value::as_str)
7287                    })
7288                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
7289                {
7290                    return Err(invalid_recorded_history(
7291                        "child_workflow_identity_mismatch",
7292                        sequence,
7293                        workflow_type
7294                            .as_deref()
7295                            .unwrap_or("one child workflow type"),
7296                        "conflicting child workflow types",
7297                        "child workflow lifecycle events at one sequence disagree on type",
7298                    ));
7299                }
7300                let mut outcomes = child_workflow_outcomes(
7301                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7302                    fallback_codec,
7303                    parent.clone(),
7304                )?;
7305                if outcomes.len() > 1 {
7306                    return Err(invalid_recorded_history(
7307                        "duplicate_child_workflow_terminal_event",
7308                        sequence,
7309                        "at most one terminal child event",
7310                        "multiple terminal child events",
7311                        "child workflow history settles one command more than once",
7312                    ));
7313                }
7314                return Ok(RecordedCommand::ChildWorkflow {
7315                    sequence,
7316                    workflow_type,
7317                    outcome: outcomes.pop(),
7318                });
7319            }
7320
7321            if !signal_wait_events.is_empty() {
7322                let opened: Vec<_> = signal_wait_events
7323                    .iter()
7324                    .copied()
7325                    .filter(|event| event.event_type == "SignalWaitOpened")
7326                    .collect();
7327                if opened.len() != 1 {
7328                    return Err(invalid_recorded_history(
7329                        "signal_wait_open_missing_or_duplicate",
7330                        sequence,
7331                        "one SignalWaitOpened event",
7332                        &format!("{} SignalWaitOpened events", opened.len()),
7333                        "signal replay requires exactly one canonical wait-open event",
7334                    ));
7335                }
7336
7337                let applied: Vec<_> = signal_wait_events
7338                    .iter()
7339                    .copied()
7340                    .filter(|event| event.event_type == "SignalApplied")
7341                    .collect();
7342                if applied.len() > 1 {
7343                    return Err(invalid_recorded_history(
7344                        "duplicate_signal_wait_apply",
7345                        sequence,
7346                        "at most one SignalApplied event",
7347                        "multiple SignalApplied events",
7348                        "signal history applies one durable wait more than once",
7349                    ));
7350                }
7351
7352                let signal_names = signal_wait_events
7353                    .iter()
7354                    .map(|event| required_signal_wait_name(event, sequence))
7355                    .collect::<Result<Vec<_>>>()?;
7356                let signal_name = signal_names
7357                    .first()
7358                    .expect("signal wait events are not empty")
7359                    .clone();
7360                if signal_names.iter().any(|candidate| candidate != &signal_name) {
7361                    return Err(invalid_recorded_history(
7362                        "signal_wait_identity_mismatch",
7363                        sequence,
7364                        &signal_name,
7365                        "conflicting signal names",
7366                        "signal wait lifecycle events at one workflow sequence disagree on identity",
7367                    ));
7368                }
7369                let value = applied
7370                    .first()
7371                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
7372                    .transpose()?;
7373                return Ok(RecordedCommand::SignalWait {
7374                    sequence,
7375                    signal_name,
7376                    value,
7377                });
7378            }
7379
7380            if !side_effect_events.is_empty() {
7381                if side_effect_events.len() != 1 {
7382                    return Err(invalid_recorded_history(
7383                        "duplicate_side_effect_record",
7384                        sequence,
7385                        "one SideEffectRecorded event",
7386                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
7387                        "side-effect history records one workflow command more than once",
7388                    ));
7389                }
7390                let event = side_effect_events[0];
7391                let result = event.payload.get("result").ok_or_else(|| {
7392                    invalid_recorded_history(
7393                        "side_effect_result_missing",
7394                        sequence,
7395                        "recorded result payload",
7396                        "missing result",
7397                        "side-effect history is missing its recorded value",
7398                    )
7399                })?;
7400                let has_published_envelope = result.as_str().is_some()
7401                    || result.as_object().is_some_and(|envelope| {
7402                        envelope.get("codec").and_then(Value::as_str).is_some()
7403                            && envelope.get("blob").and_then(Value::as_str).is_some()
7404                    });
7405                if !has_published_envelope {
7406                    return Err(invalid_recorded_history(
7407                        "side_effect_payload_malformed",
7408                        sequence,
7409                        "payload blob or {codec, blob} envelope",
7410                        &result.to_string(),
7411                        "side-effect history result does not use a published payload envelope",
7412                    ));
7413                }
7414                let codec = event
7415                    .payload
7416                    .get("payload_codec")
7417                    .and_then(Value::as_str)
7418                    .unwrap_or(fallback_codec);
7419                let value = decode_wire_avro_value(result, codec).map_err(|error| {
7420                    invalid_recorded_history(
7421                        "side_effect_payload_incompatible",
7422                        sequence,
7423                        &format!("valid {codec} payload envelope"),
7424                        &error.to_string(),
7425                        "side-effect history payload cannot be decoded with its recorded codec",
7426                    )
7427                })?;
7428                return Ok(RecordedCommand::SideEffect { sequence, value });
7429            }
7430
7431            if !version_marker_events.is_empty() {
7432                if version_marker_events.len() != 1 {
7433                    return Err(invalid_recorded_history(
7434                        "duplicate_version_marker_record",
7435                        sequence,
7436                        "one VersionMarkerRecorded event",
7437                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7438                        "version-marker history records one workflow command more than once",
7439                    ));
7440                }
7441                let payload = &version_marker_events[0].payload;
7442                let change_id = payload
7443                    .get("change_id")
7444                    .and_then(Value::as_str)
7445                    .filter(|value| !value.is_empty())
7446                    .map(str::to_string)
7447                    .ok_or_else(|| {
7448                        invalid_recorded_history(
7449                            "version_marker_field_missing",
7450                            sequence,
7451                            "non-empty change_id",
7452                            "missing or invalid change_id",
7453                            "version-marker history is missing its stable change ID",
7454                        )
7455                    })?;
7456                let version = required_version_i32(payload, "version", sequence)?;
7457                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7458                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7459                if min_supported > max_supported || version < min_supported || version > max_supported {
7460                    return Err(invalid_recorded_history(
7461                        "version_marker_history_range_invalid",
7462                        sequence,
7463                        "min_supported <= version <= max_supported",
7464                        &format!("{min_supported} <= {version} <= {max_supported}"),
7465                        "recorded version marker contains an internally incompatible range",
7466                    ));
7467                }
7468                return Ok(RecordedCommand::VersionMarker {
7469                    sequence,
7470                    change_id,
7471                    version,
7472                });
7473            }
7474
7475            let scheduled: Vec<_> = timer_events
7476                .iter()
7477                .copied()
7478                .filter(|event| event.event_type == "TimerScheduled")
7479                .collect();
7480            let fired: Vec<_> = timer_events
7481                .iter()
7482                .copied()
7483                .filter(|event| event.event_type == "TimerFired")
7484                .collect();
7485            if scheduled.len() != 1 {
7486                return Err(invalid_recorded_history(
7487                    "timer_schedule_missing_or_duplicate",
7488                    sequence,
7489                    "one TimerScheduled event",
7490                    &format!("{} TimerScheduled events", scheduled.len()),
7491                    "timer replay requires exactly one recorded schedule event",
7492                ));
7493            }
7494            if fired.len() > 1 {
7495                return Err(invalid_recorded_history(
7496                    "duplicate_timer_fire",
7497                    sequence,
7498                    "at most one TimerFired event",
7499                    "multiple TimerFired events",
7500                    "timer history contains more than one fire event for a workflow sequence",
7501                ));
7502            }
7503
7504            let scheduled = scheduled[0];
7505            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7506            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7507            if let Some(fired) = fired.first() {
7508                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7509                if fired_timer_id != timer_id {
7510                    return Err(invalid_recorded_history(
7511                        "timer_identity_mismatch",
7512                        sequence,
7513                        &timer_id,
7514                        &fired_timer_id,
7515                        "TimerFired does not correspond to the recorded TimerScheduled event",
7516                    ));
7517                }
7518                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7519                if fired_delay != delay_seconds {
7520                    return Err(invalid_recorded_history(
7521                        "timer_history_delay_mismatch",
7522                        sequence,
7523                        &delay_seconds.to_string(),
7524                        &fired_delay.to_string(),
7525                        "TimerScheduled and TimerFired record different delays",
7526                    ));
7527                }
7528            }
7529
7530            Ok(RecordedCommand::Timer {
7531                sequence,
7532                delay_seconds,
7533                fired: !fired.is_empty(),
7534            })
7535        })
7536        .collect::<Result<_>>()?;
7537
7538    let mut marker_sequences = HashMap::new();
7539    for command in &commands {
7540        if let RecordedCommand::VersionMarker {
7541            sequence,
7542            change_id,
7543            ..
7544        } = command
7545        {
7546            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7547                return Err(invalid_recorded_history(
7548                    "duplicate_version_marker",
7549                    *sequence,
7550                    &format!("one marker for change ID {change_id:?}"),
7551                    &format!("markers at sequences {first_sequence} and {sequence}"),
7552                    "workflow history contains duplicate markers for one stable change ID",
7553                ));
7554            }
7555        }
7556    }
7557
7558    Ok(commands)
7559}
7560
7561fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7562    payload
7563        .get(field)
7564        .and_then(Value::as_i64)
7565        .and_then(|value| i32::try_from(value).ok())
7566        .ok_or_else(|| {
7567            invalid_recorded_history(
7568                "version_marker_field_missing",
7569                sequence,
7570                &format!("integer {field}"),
7571                "missing or out-of-range integer",
7572                "version-marker history is missing a required integer field",
7573            )
7574        })
7575}
7576
7577fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7578    event
7579        .payload
7580        .get("sequence")
7581        .or_else(|| event.payload.get("workflow_sequence"))
7582        .or_else(|| event.raw.get("sequence"))
7583        .or_else(|| event.raw.get("workflow_sequence"))
7584        .and_then(value_as_u64)
7585}
7586
7587fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7588    matches!(
7589        event
7590            .payload
7591            .get("timer_kind")
7592            .or_else(|| event.raw.get("timer_kind"))
7593            .and_then(Value::as_str),
7594        Some("condition_timeout" | "signal_timeout")
7595    )
7596}
7597
7598fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7599    event
7600        .payload
7601        .get("signal_name")
7602        .or_else(|| event.raw.get("signal_name"))
7603        .and_then(Value::as_str)
7604        .filter(|value| !value.is_empty())
7605        .map(str::to_string)
7606        .ok_or_else(|| {
7607            invalid_recorded_history(
7608                "signal_wait_name_missing",
7609                sequence,
7610                "non-empty signal_name",
7611                &event.event_type,
7612                "canonical signal-wait history is missing its signal identity",
7613            )
7614        })
7615}
7616
7617fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7618    matches!(
7619        event.event_type.as_str(),
7620        "SignalWaitOpened" | "SignalApplied"
7621    )
7622}
7623
7624fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7625    event
7626        .payload
7627        .get(field)
7628        .and_then(Value::as_str)
7629        .filter(|value| !value.is_empty())
7630        .map(str::to_string)
7631        .ok_or_else(|| {
7632            invalid_recorded_history(
7633                "timer_history_field_missing",
7634                sequence,
7635                field,
7636                &event.event_type,
7637                "timer history is missing a required identity field",
7638            )
7639        })
7640}
7641
7642fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7643    event
7644        .payload
7645        .get(field)
7646        .and_then(value_as_u64)
7647        .ok_or_else(|| {
7648            invalid_recorded_history(
7649                "timer_history_field_missing",
7650                sequence,
7651                field,
7652                &event.event_type,
7653                "timer history is missing a required numeric field",
7654            )
7655        })
7656}
7657
7658fn invalid_recorded_history(
7659    reason: &str,
7660    sequence: u64,
7661    expected: &str,
7662    actual: &str,
7663    message: &str,
7664) -> Error {
7665    Error::NonDeterministicReplay(ReplayFailure::new(
7666        reason,
7667        Some(sequence),
7668        Some(expected.to_string()),
7669        Some(actual.to_string()),
7670        message,
7671    ))
7672}
7673
7674type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7675
7676fn activity_outcome(
7677    event: &HistoryEvent,
7678    fallback_codec: &str,
7679    recorded_activity_type: Option<String>,
7680) -> Result<ActivityOutcome> {
7681    if event.event_type == "ActivityCompleted" {
7682        let codec = event
7683            .payload
7684            .get("payload_codec")
7685            .and_then(Value::as_str)
7686            .unwrap_or(fallback_codec);
7687        return Ok(Ok(decode_wire_avro_value(
7688            event.payload.get("result").unwrap_or(&Value::Null),
7689            codec,
7690        )?));
7691    }
7692
7693    let payload = &event.payload;
7694    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7695        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7696        "ActivityCancelled" => (
7697            ActivityFailureKind::Cancelled,
7698            "cancelled",
7699            "activity was cancelled",
7700        ),
7701        "ActivityTimedOut" => (
7702            ActivityFailureKind::TimedOut,
7703            "timeout",
7704            "activity timed out",
7705        ),
7706        _ => unreachable!("activity_outcome is called only for terminal activity events"),
7707    };
7708    let exception = payload
7709        .get("exception")
7710        .filter(|value| !value.is_null())
7711        .cloned();
7712    let failure_category = payload_string(payload, "failure_category");
7713    let timeout_kind = payload_string(payload, "timeout_kind");
7714    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7715        ActivityFailureKind::Failed => failure_category
7716            .clone()
7717            .unwrap_or_else(|| fallback_reason.to_string()),
7718        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7719        ActivityFailureKind::TimedOut => timeout_kind
7720            .clone()
7721            .unwrap_or_else(|| fallback_reason.to_string()),
7722    });
7723    let message = payload_string(payload, "message")
7724        .or_else(|| {
7725            exception
7726                .as_ref()
7727                .and_then(|value| payload_string(value, "message"))
7728        })
7729        .unwrap_or_else(|| fallback_message.to_string());
7730
7731    Ok(Err(ActivityFailure {
7732        kind,
7733        reason,
7734        message,
7735        activity_execution_id: payload_string(payload, "activity_execution_id"),
7736        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7737        activity_type: payload_string(payload, "activity_type")
7738            .or_else(|| payload_string(payload, "activity_name"))
7739            .or(recorded_activity_type),
7740        activity_class: payload_string(payload, "activity_class"),
7741        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7742        failure_id: payload_string(payload, "failure_id"),
7743        failure_category,
7744        timeout_kind,
7745        non_retryable: payload
7746            .get("non_retryable")
7747            .and_then(Value::as_bool)
7748            .unwrap_or(false),
7749        exception_type: payload_string(payload, "exception_type").or_else(|| {
7750            exception
7751                .as_ref()
7752                .and_then(|value| payload_string(value, "type"))
7753        }),
7754        exception_class: payload_string(payload, "exception_class").or_else(|| {
7755            exception
7756                .as_ref()
7757                .and_then(|value| payload_string(value, "class"))
7758        }),
7759        code: payload
7760            .get("code")
7761            .filter(|value| !value.is_null())
7762            .cloned(),
7763        exception,
7764    }))
7765}
7766
7767type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7768
7769fn child_workflow_outcomes(
7770    events: &[HistoryEvent],
7771    fallback_codec: &str,
7772    parent: WorkflowIdentity,
7773) -> Result<Vec<ChildWorkflowOutcome>> {
7774    let mut outcomes = Vec::new();
7775
7776    for event in events {
7777        let kind = match event.event_type.as_str() {
7778            "ChildRunCompleted" => None,
7779            "ChildRunFailed" => Some((
7780                ChildWorkflowFailureKind::Failed,
7781                "child_workflow",
7782                "child workflow failed",
7783            )),
7784            "ChildRunCancelled" => Some((
7785                ChildWorkflowFailureKind::Cancelled,
7786                "cancelled",
7787                "child workflow was cancelled",
7788            )),
7789            "ChildRunTerminated" => Some((
7790                ChildWorkflowFailureKind::Terminated,
7791                "terminated",
7792                "child workflow was terminated",
7793            )),
7794            _ => continue,
7795        };
7796        let payload = &event.payload;
7797        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7798        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7799        let child_workflow_type = payload_string(payload, "child_workflow_type");
7800
7801        if let Some((kind, reason, fallback_message)) = kind {
7802            let exception = payload
7803                .get("exception")
7804                .filter(|value| !value.is_null())
7805                .cloned();
7806            let message = payload_string(payload, "message")
7807                .or_else(|| {
7808                    exception
7809                        .as_ref()
7810                        .and_then(|value| payload_string(value, "message"))
7811                })
7812                .unwrap_or_else(|| fallback_message.to_string());
7813            let exception_type = payload_string(payload, "exception_type").or_else(|| {
7814                exception
7815                    .as_ref()
7816                    .and_then(|value| payload_string(value, "type"))
7817            });
7818            let exception_class = payload_string(payload, "exception_class").or_else(|| {
7819                exception
7820                    .as_ref()
7821                    .and_then(|value| payload_string(value, "class"))
7822            });
7823            outcomes.push(Err(ChildWorkflowFailure {
7824                kind,
7825                reason: reason.to_string(),
7826                message,
7827                parent_workflow_id: parent.workflow_id.clone(),
7828                parent_workflow_run_id: parent.run_id.clone(),
7829                child_workflow_id,
7830                child_workflow_run_id,
7831                child_workflow_type,
7832                failure_id: payload_string(payload, "failure_id"),
7833                failure_category: payload_string(payload, "failure_category"),
7834                exception_type,
7835                exception_class,
7836                non_retryable: payload
7837                    .get("non_retryable")
7838                    .and_then(Value::as_bool)
7839                    .unwrap_or(false),
7840                code: payload
7841                    .get("code")
7842                    .filter(|value| !value.is_null())
7843                    .cloned(),
7844                exception,
7845            }));
7846            continue;
7847        }
7848
7849        let codec = payload
7850            .get("payload_codec")
7851            .and_then(Value::as_str)
7852            .unwrap_or(fallback_codec);
7853        let result = payload
7854            .get("result")
7855            .or_else(|| payload.get("output"))
7856            .unwrap_or(&Value::Null);
7857        outcomes.push(Ok(ChildWorkflowAvroResult {
7858            parent: parent.clone(),
7859            child: WorkflowIdentity {
7860                workflow_id: child_workflow_id,
7861                run_id: child_workflow_run_id,
7862            },
7863            child_workflow_type,
7864            result: decode_wire_avro_value(result, codec)?,
7865        }));
7866    }
7867
7868    Ok(outcomes)
7869}
7870
7871fn payload_string(payload: &Value, key: &str) -> Option<String> {
7872    payload
7873        .get(key)
7874        .and_then(Value::as_str)
7875        .filter(|value| !value.is_empty())
7876        .map(str::to_string)
7877}
7878
7879fn workflow_failure_command(error: &Error) -> Value {
7880    let (exception_type, exception_class, properties) = match error {
7881        Error::ActivityFailed(failure) => (
7882            match failure.kind {
7883                ActivityFailureKind::Failed => "ActivityFailed",
7884                ActivityFailureKind::Cancelled => "ActivityCancelled",
7885                ActivityFailureKind::TimedOut => "ActivityTimedOut",
7886            },
7887            "durable_workflow::ActivityFailure",
7888            json!({
7889                "reason": failure.reason,
7890                "activity_execution_id": failure.activity_execution_id,
7891                "activity_attempt_id": failure.activity_attempt_id,
7892                "activity_type": failure.activity_type,
7893                "activity_class": failure.activity_class,
7894                "attempt_number": failure.attempt_number,
7895                "failure_id": failure.failure_id,
7896                "failure_category": failure.failure_category,
7897                "timeout_kind": failure.timeout_kind,
7898                "activity_non_retryable": failure.non_retryable,
7899                "activity_exception_type": failure.exception_type,
7900                "activity_exception_class": failure.exception_class,
7901                "activity_code": failure.code,
7902                "activity_exception": failure.exception,
7903            }),
7904        ),
7905        Error::ChildWorkflowFailed(failure) => (
7906            match failure.kind {
7907                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7908                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7909                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7910            },
7911            "durable_workflow::ChildWorkflowFailure",
7912            json!({
7913                "reason": failure.reason,
7914                "parent_workflow_id": failure.parent_workflow_id,
7915                "parent_workflow_run_id": failure.parent_workflow_run_id,
7916                "child_workflow_id": failure.child_workflow_id,
7917                "child_workflow_run_id": failure.child_workflow_run_id,
7918                "child_workflow_type": failure.child_workflow_type,
7919                "failure_id": failure.failure_id,
7920                "failure_category": failure.failure_category,
7921                "child_exception_type": failure.exception_type,
7922                "child_exception_class": failure.exception_class,
7923                "child_non_retryable": failure.non_retryable,
7924                "child_code": failure.code,
7925                "child_exception": failure.exception,
7926            }),
7927        ),
7928        Error::NonDeterministicReplay(_) => (
7929            "NonDeterministicReplay",
7930            "durable_workflow::Error",
7931            Value::Null,
7932        ),
7933        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7934    };
7935    let non_retryable = match error {
7936        Error::ActivityFailed(failure) => failure.non_retryable,
7937        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7938        Error::NonDeterministicReplay(_) => true,
7939        _ => false,
7940    };
7941
7942    json!({
7943        "type": "fail_workflow",
7944        "message": error.to_string(),
7945        "exception_type": exception_type,
7946        "exception_class": exception_class,
7947        "non_retryable": non_retryable,
7948        "exception": {
7949            "type": exception_type,
7950            "class": exception_class,
7951            "message": error.to_string(),
7952            "properties": properties,
7953        }
7954    })
7955}
7956
7957fn workflow_task_integrity_error(error: &Error) -> bool {
7958    matches!(
7959        error,
7960        Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7961    )
7962}
7963
7964fn decode_signal_event_arguments(
7965    event: &HistoryEvent,
7966    fallback_codec: &str,
7967) -> Result<Vec<AvroValue>> {
7968    let codec = event
7969        .payload
7970        .get("payload_codec")
7971        .and_then(Value::as_str)
7972        .unwrap_or(fallback_codec);
7973    let raw = event
7974        .payload
7975        .get("value")
7976        .or_else(|| event.payload.get("input"))
7977        .or_else(|| event.payload.get("arguments"));
7978    let decoded = match raw.filter(|value| !value.is_null()) {
7979        Some(value) => decode_wire_avro_value(value, codec)?,
7980        None => AvroValue::Array(Vec::new()),
7981    };
7982    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
7983        unreachable!("normalize_avro_arguments always returns an array");
7984    };
7985    Ok(arguments)
7986}
7987
7988fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
7989    let Some(export_events) = task
7990        .history_export
7991        .as_ref()
7992        .and_then(|export| export.get("history_events"))
7993        .and_then(Value::as_array)
7994    else {
7995        return Ok(());
7996    };
7997
7998    if export_events.len() > task.history_events.len() {
7999        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8000    }
8001
8002    Ok(())
8003}
8004
8005fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8006    let Some(export) = task.history_export.as_ref() else {
8007        return Ok(());
8008    };
8009    let signals = export
8010        .get("signals")
8011        .and_then(Value::as_array)
8012        .cloned()
8013        .unwrap_or_default();
8014    let activities = export
8015        .get("activities")
8016        .and_then(Value::as_array)
8017        .cloned()
8018        .unwrap_or_default();
8019    let export_codec = export
8020        .get("payloads")
8021        .and_then(|payloads| payloads.get("codec"))
8022        .and_then(Value::as_str)
8023        .unwrap_or(&task.payload_codec)
8024        .to_string();
8025    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8026
8027    for event in &mut task.history_events {
8028        if event.event_type == "ActivityCompleted" {
8029            let sequence = event
8030                .payload
8031                .get("sequence")
8032                .or_else(|| event.payload.get("workflow_sequence"))
8033                .and_then(value_as_u64);
8034            let Some(activity) = sequence.and_then(|sequence| {
8035                activities.iter().find(|activity| {
8036                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8037                })
8038            }) else {
8039                continue;
8040            };
8041            let Some(payload) = event.payload.as_object_mut() else {
8042                continue;
8043            };
8044            if missing_payload(payload.get("result")) {
8045                if let Some(result) = activity
8046                    .get("result")
8047                    .filter(|value| !missing_payload(Some(value)))
8048                {
8049                    payload.insert("result".to_string(), result.clone());
8050                }
8051            }
8052            for field in ["payload_codec", "activity_type"] {
8053                if payload
8054                    .get(field)
8055                    .and_then(Value::as_str)
8056                    .unwrap_or_default()
8057                    .is_empty()
8058                {
8059                    if let Some(value) = activity.get(field) {
8060                        payload.insert(field.to_string(), value.clone());
8061                    }
8062                }
8063            }
8064            continue;
8065        }
8066
8067        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8068            continue;
8069        }
8070        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8071        let command_id = event
8072            .payload
8073            .get("workflow_command_id")
8074            .or_else(|| event.raw.get("workflow_command_id"))
8075            .and_then(Value::as_str);
8076        let signal_name = event
8077            .payload
8078            .get("signal_name")
8079            .and_then(Value::as_str)
8080            .unwrap_or_default()
8081            .to_string();
8082        let matched = signals
8083            .iter()
8084            .find(|signal| {
8085                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8086            })
8087            .or_else(|| {
8088                signals.iter().find(|signal| {
8089                    command_id.is_some()
8090                        && signal.get("command_id").and_then(Value::as_str) == command_id
8091                })
8092            })
8093            .or_else(|| {
8094                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8095                let signal = signals
8096                    .iter()
8097                    .filter(|signal| {
8098                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8099                    })
8100                    .nth(*offset);
8101                if signal.is_some() {
8102                    *offset += 1;
8103                }
8104                signal
8105            });
8106        let Some(signal) = matched else {
8107            continue;
8108        };
8109        let signal_codec = signal
8110            .get("payload_codec")
8111            .and_then(Value::as_str)
8112            .unwrap_or(&export_codec);
8113        let Some(payload) = event.payload.as_object_mut() else {
8114            continue;
8115        };
8116        if missing_payload(payload.get("arguments")) {
8117            if let Some(arguments) = signal
8118                .get("arguments")
8119                .filter(|value| !missing_payload(Some(value)))
8120            {
8121                let envelope = match arguments {
8122                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8123                    other => other.clone(),
8124                };
8125                payload.insert("arguments".to_string(), envelope);
8126            }
8127        }
8128        if payload
8129            .get("payload_codec")
8130            .and_then(Value::as_str)
8131            .unwrap_or_default()
8132            .is_empty()
8133        {
8134            payload.insert("payload_codec".to_string(), json!(signal_codec));
8135        }
8136    }
8137
8138    Ok(())
8139}
8140
8141fn missing_payload(value: Option<&Value>) -> bool {
8142    match value {
8143        None | Some(Value::Null) => true,
8144        Some(Value::String(value)) => value.is_empty(),
8145        Some(_) => false,
8146    }
8147}
8148
8149fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8150    let export_signals = task
8151        .history_export
8152        .as_ref()
8153        .and_then(|export| export.get("signals"))
8154        .and_then(Value::as_array)
8155        .cloned()
8156        .unwrap_or_default();
8157    let export_codec = task
8158        .history_export
8159        .as_ref()
8160        .and_then(|export| export.get("payloads"))
8161        .and_then(|payloads| payloads.get("codec"))
8162        .and_then(Value::as_str)
8163        .unwrap_or(&task.payload_codec);
8164    let mut name_offsets: HashMap<String, usize> = HashMap::new();
8165    let mut signals = Vec::new();
8166
8167    for event in &task.history_events {
8168        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8169            continue;
8170        }
8171
8172        let name = event
8173            .payload
8174            .get("signal_name")
8175            .and_then(Value::as_str)
8176            .unwrap_or_default();
8177        if name.is_empty() {
8178            continue;
8179        }
8180        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8181        let command_id = event
8182            .payload
8183            .get("workflow_command_id")
8184            .or_else(|| event.raw.get("workflow_command_id"))
8185            .and_then(Value::as_str);
8186        let matched_export = export_signals
8187            .iter()
8188            .find(|candidate| {
8189                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8190            })
8191            .or_else(|| {
8192                export_signals.iter().find(|candidate| {
8193                    command_id.is_some()
8194                        && candidate.get("command_id").and_then(Value::as_str) == command_id
8195                })
8196            })
8197            .or_else(|| {
8198                let offset = name_offsets.entry(name.to_string()).or_default();
8199                let candidate = export_signals
8200                    .iter()
8201                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8202                    .nth(*offset);
8203                if candidate.is_some() {
8204                    *offset += 1;
8205                }
8206                candidate
8207            });
8208        let codec = event
8209            .payload
8210            .get("payload_codec")
8211            .and_then(Value::as_str)
8212            .or_else(|| {
8213                matched_export
8214                    .and_then(|signal| signal.get("payload_codec"))
8215                    .and_then(Value::as_str)
8216            })
8217            .unwrap_or(export_codec);
8218        let raw_arguments = event
8219            .payload
8220            .get("value")
8221            .or_else(|| event.payload.get("input"))
8222            .or_else(|| event.payload.get("arguments"))
8223            .filter(|value| !value.is_null())
8224            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8225        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8226        let workflow_sequence = event
8227            .payload
8228            .get("workflow_sequence")
8229            .and_then(value_as_u64)
8230            .or_else(|| {
8231                matched_export
8232                    .and_then(|signal| signal.get("workflow_sequence"))
8233                    .and_then(value_as_u64)
8234            });
8235
8236        signals.push(QuerySignal {
8237            id: signal_id.map(str::to_string).or_else(|| {
8238                matched_export
8239                    .and_then(|signal| signal.get("id"))
8240                    .and_then(Value::as_str)
8241                    .map(str::to_string)
8242            }),
8243            name: name.to_string(),
8244            arguments,
8245            avro_arguments,
8246            workflow_sequence,
8247        });
8248    }
8249
8250    if signals.is_empty() {
8251        for signal in export_signals {
8252            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8253                continue;
8254            }
8255            let Some(name) = signal.get("name").and_then(Value::as_str) else {
8256                continue;
8257            };
8258            let codec = signal
8259                .get("payload_codec")
8260                .and_then(Value::as_str)
8261                .unwrap_or(export_codec);
8262            let (arguments, avro_arguments) =
8263                decode_query_signal_arguments(signal.get("arguments"), codec)?;
8264            signals.push(QuerySignal {
8265                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8266                name: name.to_string(),
8267                arguments,
8268                avro_arguments,
8269                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8270            });
8271        }
8272        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8273    }
8274
8275    Ok(signals)
8276}
8277
8278fn decode_query_signal_arguments(
8279    raw: Option<&Value>,
8280    codec: &str,
8281) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8282    let decoded = match raw.filter(|value| !value.is_null()) {
8283        Some(value) => decode_wire_avro_value(value, codec)?,
8284        None => AvroValue::Array(Vec::new()),
8285    };
8286    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8287        unreachable!("normalize_avro_arguments always returns an array");
8288    };
8289    let arguments = avro_arguments
8290        .iter()
8291        .cloned()
8292        .map(AvroValue::into_json)
8293        .collect::<Result<Vec<_>>>()?;
8294    Ok((arguments, avro_arguments))
8295}
8296
8297fn value_as_u64(value: &Value) -> Option<u64> {
8298    value
8299        .as_u64()
8300        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8301}
8302
8303#[cfg(test)]
8304mod tests {
8305    use super::*;
8306    use std::{
8307        io::{Read, Write},
8308        net::{SocketAddr, TcpListener, TcpStream},
8309        sync::atomic::AtomicUsize,
8310        thread,
8311    };
8312
8313    #[test]
8314    fn client_builder_rejects_the_sdk_owned_api_suffix() {
8315        for base_url in [
8316            "http://127.0.0.1:8080/api",
8317            "http://localhost:8080/api/",
8318            "https://runtime.example.test/namespaces/orders/api",
8319        ] {
8320            let error = Client::builder(base_url)
8321                .build()
8322                .expect_err("SDK-owned /api suffix must be rejected during build");
8323
8324            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8325            assert!(
8326                error.to_string().contains("SDK appends /api automatically"),
8327                "the validation error must explain how to fix the endpoint"
8328            );
8329        }
8330    }
8331
8332    #[test]
8333    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8334        for (base_url, expected) in [
8335            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8336            (
8337                "http://localhost:8080/durable-workflow/",
8338                "http://localhost:8080/durable-workflow",
8339            ),
8340            (
8341                "https://runtime.example.test/namespaces/orders",
8342                "https://runtime.example.test/namespaces/orders",
8343            ),
8344            (
8345                "https://runtime.example.test/gateway/api/namespaces/orders",
8346                "https://runtime.example.test/gateway/api/namespaces/orders",
8347            ),
8348            (
8349                "https://api.example.test/runtime/orders/",
8350                "https://api.example.test/runtime/orders",
8351            ),
8352        ] {
8353            let client = Client::builder(base_url)
8354                .build()
8355                .expect("Server and Cloud runtime base URL must remain valid");
8356
8357            assert_eq!(client.base_url, expected);
8358        }
8359    }
8360
8361    fn typed_fidelity_probe() -> AvroValue {
8362        AvroValue::Map(BTreeMap::from([
8363            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8364            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8365            (
8366                "numeric".to_string(),
8367                AvroValue::Map(BTreeMap::from([
8368                    ("0".to_string(), AvroValue::String("zero".to_string())),
8369                    ("1".to_string(), AvroValue::String("one".to_string())),
8370                ])),
8371            ),
8372            (
8373                "nested".to_string(),
8374                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8375                    "enabled".to_string(),
8376                    AvroValue::Boolean(true),
8377                )]))]),
8378            ),
8379            (
8380                "projection_collisions".to_string(),
8381                AvroValue::Array(projection_collision_probe()),
8382            ),
8383        ]))
8384    }
8385
8386    fn projection_collision_probe() -> Vec<AvroValue> {
8387        vec![
8388            AvroValue::Map(BTreeMap::from([
8389                ("$type".to_string(), AvroValue::String("bytes".to_string())),
8390                (
8391                    "base64".to_string(),
8392                    AvroValue::String("ordinary user text".to_string()),
8393                ),
8394            ])),
8395            AvroValue::Map(BTreeMap::from([
8396                ("$type".to_string(), AvroValue::String("map".to_string())),
8397                (
8398                    "entries".to_string(),
8399                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8400                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
8401                        (
8402                            "value".to_string(),
8403                            AvroValue::String("user map".to_string()),
8404                        ),
8405                    ]))]),
8406                ),
8407            ])),
8408        ]
8409    }
8410
8411    #[derive(Clone, Debug, Default, PartialEq)]
8412    struct ReplayCounterState {
8413        loaded: Option<String>,
8414        count: i64,
8415        finished: bool,
8416    }
8417
8418    fn replay_counter_worker() -> Worker {
8419        let client = Client::new("http://127.0.0.1:8080").expect("client");
8420        let mut worker = Worker::new(client, "rust-workers");
8421        worker.register_replayed_workflow(
8422            "replay-counter",
8423            ReplayCounterState::default,
8424            |ctx, _input, state| async move {
8425                let loaded = ctx.activity("load-counter", json!([])).await?;
8426                state.update(|current| {
8427                    current.loaded = loaded.as_str().map(str::to_string);
8428                })?;
8429                for _ in 0..2 {
8430                    let signal = ctx.wait_signal("increment").await?;
8431                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8432                    state.update(|current| current.count += amount)?;
8433                }
8434                state.update(|current| current.finished = true)?;
8435                state.read(|current| Ok(json!(current.count)))?
8436            },
8437        );
8438        worker.register_replayed_query::<ReplayCounterState, _, _>(
8439            "replay-counter",
8440            "current",
8441            |_ctx, state, _args| async move {
8442                Ok(json!({
8443                    "loaded": state.loaded,
8444                    "count": state.count,
8445                    "finished": state.finished,
8446                }))
8447            },
8448        );
8449        worker.register_replayed_query::<ReplayCounterState, _, _>(
8450            "replay-counter",
8451            "detached-mutation",
8452            |_ctx, state, _args| async move {
8453                let mut detached = (*state).clone();
8454                detached.count = 999;
8455                Ok(json!(detached.count))
8456            },
8457        );
8458        worker.register_replayed_query::<ReplayCounterState, _, _>(
8459            "replay-counter",
8460            "failed-mutation",
8461            |_ctx, state, _args| async move {
8462                let mut detached = (*state).clone();
8463                detached.count = 999;
8464                Err(Error::WorkerLoop("query refused".to_string()))
8465            },
8466        );
8467        worker
8468    }
8469
8470    fn replay_counter_query(
8471        query_name: &str,
8472        history_events: Value,
8473        run_status: &str,
8474    ) -> QueryTask {
8475        serde_json::from_value(json!({
8476            "query_task_id": format!("query-{query_name}"),
8477            "workflow_type": "replay-counter",
8478            "query_name": query_name,
8479            "payload_codec": "json",
8480            "workflow_arguments": {"codec": "json", "blob": "[]"},
8481            "query_arguments": {"codec": "json", "blob": "[]"},
8482            "history_events": history_events,
8483            "run_status": run_status,
8484        }))
8485        .expect("query task")
8486    }
8487
8488    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8489        workflow_context_with_codec(history, JSON_CODEC)
8490    }
8491
8492    fn workflow_context_with_codec(
8493        history: Vec<HistoryEvent>,
8494        payload_codec: &str,
8495    ) -> WorkflowContext {
8496        WorkflowContext {
8497            state: Arc::new(Mutex::new(
8498                WorkflowState::new_with_identity(
8499                    history,
8500                    None,
8501                    None,
8502                    "rust-workers".to_string(),
8503                    payload_codec.to_string(),
8504                    None,
8505                )
8506                .expect("valid workflow history"),
8507            )),
8508        }
8509    }
8510
8511    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8512        HistoryEvent {
8513            event_type: event_type.to_string(),
8514            payload,
8515            raw: HashMap::new(),
8516        }
8517    }
8518
8519    fn workflow_task(
8520        workflow_type: &str,
8521        history_events: Vec<HistoryEvent>,
8522        payload_codec: &str,
8523    ) -> WorkflowTask {
8524        WorkflowTask {
8525            task_id: format!("wft-{workflow_type}"),
8526            workflow_id: Some(format!("wf-{workflow_type}")),
8527            run_id: Some(format!("run-{workflow_type}")),
8528            workflow_type: workflow_type.to_string(),
8529            payload_codec: payload_codec.to_string(),
8530            arguments: Some(
8531                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8532            ),
8533            total_history_events: Some(history_events.len() as u64),
8534            history_size_bytes: None,
8535            continue_as_new_recommended: None,
8536            history_budget_pressure: None,
8537            history_events,
8538            next_history_page_token: None,
8539            workflow_task_attempt: 1,
8540            workflow_signal_id: None,
8541            signal_name: None,
8542            signal_arguments: None,
8543            workflow_update_id: None,
8544            update_name: None,
8545            lease_owner: Some("rust-worker".to_string()),
8546        }
8547    }
8548
8549    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8550    struct SideEffectProbe {
8551        request_id: String,
8552        attempt: u32,
8553    }
8554
8555    #[test]
8556    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8557        let calls = AtomicUsize::new(0);
8558        let ctx = workflow_context(Vec::new());
8559        let value = ctx
8560            .side_effect(|| {
8561                calls.fetch_add(1, Ordering::SeqCst);
8562                SideEffectProbe {
8563                    request_id: "request-42".to_string(),
8564                    attempt: 3,
8565                }
8566            })
8567            .expect("first side effect");
8568        assert_eq!(value.attempt, 3);
8569        assert_eq!(calls.load(Ordering::SeqCst), 1);
8570        let commands = ctx.take_commands().expect("commands");
8571        assert_eq!(commands.len(), 1);
8572        assert_eq!(commands[0]["type"], "record_side_effect");
8573        assert_eq!(
8574            decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8575            serde_json::to_value(&value).expect("value")
8576        );
8577
8578        let replay = workflow_context(vec![history_event(
8579            "SideEffectRecorded",
8580            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8581        )]);
8582        let replayed: SideEffectProbe = replay
8583            .side_effect(|| {
8584                calls.fetch_add(1, Ordering::SeqCst);
8585                panic!("committed side-effect callbacks must not run during replay")
8586            })
8587            .expect("replayed side effect");
8588        assert_eq!(replayed, value);
8589        assert_eq!(calls.load(Ordering::SeqCst), 1);
8590        assert!(replay.take_commands().expect("commands").is_empty());
8591        replay.ensure_history_consumed().expect("history consumed");
8592    }
8593
8594    #[test]
8595    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8596        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8597        let value = ctx
8598            .side_effect(|| SideEffectProbe {
8599                request_id: "avro-request".to_string(),
8600                attempt: 1,
8601            })
8602            .expect("Avro side effect");
8603        let uuid = ctx.uuid_v4().expect("deterministic UUID");
8604        let commands = ctx.take_commands().expect("commands");
8605        assert_eq!(commands.len(), 2);
8606        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8607        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8608        assert_eq!(
8609            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8610            serde_json::to_value(&value).expect("value")
8611        );
8612
8613        let replay = workflow_context_with_codec(
8614            vec![
8615                history_event(
8616                    "SideEffectRecorded",
8617                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8618                ),
8619                history_event(
8620                    "SideEffectRecorded",
8621                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8622                ),
8623            ],
8624            DEFAULT_CODEC,
8625        );
8626        let replayed: SideEffectProbe = replay
8627            .side_effect(|| panic!("Avro callback must not run"))
8628            .expect("replayed Avro value");
8629        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8630        assert_eq!(replayed, value);
8631        assert_eq!(replayed_uuid, uuid);
8632        assert!(replay.take_commands().expect("commands").is_empty());
8633    }
8634
8635    #[test]
8636    fn typed_side_effect_replay_preserves_bytes_and_maps() {
8637        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8638        let value = ctx
8639            .side_effect_avro_value(typed_fidelity_probe)
8640            .expect("typed side effect");
8641        let commands = ctx.take_commands().expect("side-effect command");
8642        assert_eq!(
8643            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8644                .expect("recorded side effect"),
8645            value
8646        );
8647
8648        let replay = workflow_context_with_codec(
8649            vec![history_event(
8650                "SideEffectRecorded",
8651                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8652            )],
8653            DEFAULT_CODEC,
8654        );
8655        assert_eq!(
8656            replay
8657                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8658                .expect("replayed typed side effect"),
8659            value
8660        );
8661    }
8662
8663    #[test]
8664    fn ordered_side_effects_share_the_durable_command_stream() {
8665        let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8666        let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8667        let ctx = workflow_context(vec![
8668            history_event(
8669                "SideEffectRecorded",
8670                json!({"sequence": 1, "result": first}),
8671            ),
8672            history_event(
8673                "SideEffectRecorded",
8674                json!({"sequence": 2, "result": second}),
8675            ),
8676        ]);
8677        let first: String = ctx
8678            .side_effect(|| panic!("first callback must not run"))
8679            .expect("first replay");
8680        let second: i32 = ctx
8681            .side_effect(|| panic!("second callback must not run"))
8682            .expect("second replay");
8683        assert_eq!(first, "first");
8684        assert_eq!(second, 29);
8685        ctx.ensure_history_consumed().expect("ordered history");
8686
8687        let reordered = workflow_context(vec![history_event(
8688            "VersionMarkerRecorded",
8689            json!({
8690                "sequence": 1,
8691                "change_id": "before-side-effect",
8692                "version": 1,
8693                "min_supported": 1,
8694                "max_supported": 1,
8695            }),
8696        )]);
8697        let error = reordered
8698            .side_effect(|| "new".to_string())
8699            .expect_err("command reordering must fail");
8700        assert!(matches!(
8701            error,
8702            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8703                if reason == "recorded_command_mismatch"
8704        ));
8705    }
8706
8707    #[test]
8708    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8709        let ctx = workflow_context(Vec::new());
8710        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8711        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8712        assert!(ctx.patched("new-search").expect("patch"));
8713        ctx.deprecate_patch("new-search").expect("deprecate patch");
8714        let commands = ctx.take_commands().expect("commands");
8715        assert_eq!(commands.len(), 2);
8716        assert_eq!(commands[0]["type"], "record_version_marker");
8717        assert_eq!(commands[0]["version"], 2);
8718        assert_eq!(commands[1]["change_id"], "new-search");
8719
8720        let replay = workflow_context(vec![history_event(
8721            "VersionMarkerRecorded",
8722            json!({
8723                "sequence": 1,
8724                "change_id": "checkout-v2",
8725                "version": 2,
8726                "min_supported": 1,
8727                "max_supported": 2,
8728            }),
8729        )]);
8730        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8731        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8732        assert!(replay.take_commands().expect("commands").is_empty());
8733        replay.ensure_history_consumed().expect("history consumed");
8734    }
8735
8736    #[test]
8737    fn version_markers_reject_incompatible_or_malformed_history() {
8738        let incompatible = workflow_context(vec![history_event(
8739            "VersionMarkerRecorded",
8740            json!({
8741                "sequence": 1,
8742                "change_id": "checkout-v2",
8743                "version": 1,
8744                "min_supported": 1,
8745                "max_supported": 2,
8746            }),
8747        )]);
8748        let error = incompatible
8749            .get_version("checkout-v2", 2, 3)
8750            .expect_err("old version is unsupported");
8751        assert!(matches!(
8752            error,
8753            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8754                if reason == "version_marker_incompatible_range"
8755        ));
8756
8757        for (history, reason) in [
8758            (
8759                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8760                "side_effect_result_missing",
8761            ),
8762            (
8763                vec![history_event(
8764                    "SideEffectRecorded",
8765                    json!({
8766                        "sequence": 1,
8767                        "result": {"codec": "avro", "blob": "not-base64"},
8768                    }),
8769                )],
8770                "side_effect_payload_incompatible",
8771            ),
8772            (
8773                vec![history_event(
8774                    "SideEffectRecorded",
8775                    json!({"sequence": 1, "result": {"unwrapped": true}}),
8776                )],
8777                "side_effect_payload_malformed",
8778            ),
8779            (
8780                vec![history_event(
8781                    "VersionMarkerRecorded",
8782                    json!({
8783                        "sequence": 1,
8784                        "change_id": "change",
8785                        "version": 1,
8786                        "min_supported": 2,
8787                        "max_supported": 1,
8788                    }),
8789                )],
8790                "version_marker_history_range_invalid",
8791            ),
8792        ] {
8793            let error = WorkflowState::new(
8794                history,
8795                "rust-workers".to_string(),
8796                JSON_CODEC.to_string(),
8797                None,
8798            )
8799            .expect_err("malformed history must fail");
8800            assert!(matches!(
8801                error,
8802                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8803                    if actual == reason
8804            ));
8805        }
8806    }
8807
8808    #[test]
8809    fn duplicate_side_effects_and_version_markers_are_rejected() {
8810        let duplicate_side_effect = WorkflowState::new(
8811            vec![
8812                history_event(
8813                    "SideEffectRecorded",
8814                    json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8815                ),
8816                history_event(
8817                    "SideEffectRecorded",
8818                    json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8819                ),
8820            ],
8821            "rust-workers".to_string(),
8822            JSON_CODEC.to_string(),
8823            None,
8824        )
8825        .expect_err("duplicate side effect");
8826        assert!(matches!(
8827            duplicate_side_effect,
8828            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8829                if reason == "duplicate_side_effect_record"
8830        ));
8831
8832        let marker = |sequence| {
8833            history_event(
8834                "VersionMarkerRecorded",
8835                json!({
8836                    "sequence": sequence,
8837                    "change_id": "same-change",
8838                    "version": 1,
8839                    "min_supported": 1,
8840                    "max_supported": 1,
8841                }),
8842            )
8843        };
8844        let duplicate_marker = WorkflowState::new(
8845            vec![marker(1), marker(3)],
8846            "rust-workers".to_string(),
8847            JSON_CODEC.to_string(),
8848            None,
8849        )
8850        .expect_err("duplicate marker");
8851        assert!(matches!(
8852            duplicate_marker,
8853            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8854                if reason == "duplicate_version_marker"
8855        ));
8856    }
8857
8858    #[test]
8859    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8860        fn worker(calls: Arc<AtomicUsize>) -> Worker {
8861            let client = Client::new("http://127.0.0.1:8080").expect("client");
8862            let mut worker = Worker::new(client, "rust-workers");
8863            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8864                let calls = Arc::clone(&calls);
8865                async move {
8866                    let captured = ctx.side_effect(|| {
8867                        calls.fetch_add(1, Ordering::SeqCst);
8868                        "captured-once".to_string()
8869                    })?;
8870                    let version = ctx.get_version("cold-restart", 1, 2)?;
8871                    Ok(json!({"captured": captured, "version": version}))
8872                }
8873            });
8874            worker
8875        }
8876
8877        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8878            WorkflowTask {
8879                task_id: "wft-side-effect-version".to_string(),
8880                workflow_id: Some("wf-side-effect-version".to_string()),
8881                run_id: Some("run-side-effect-version".to_string()),
8882                workflow_type: "rust.side-effect-version".to_string(),
8883                payload_codec: JSON_CODEC.to_string(),
8884                arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8885                history_events,
8886                total_history_events: None,
8887                history_size_bytes: None,
8888                continue_as_new_recommended: None,
8889                history_budget_pressure: None,
8890                next_history_page_token: None,
8891                workflow_task_attempt: 1,
8892                workflow_signal_id: None,
8893                signal_name: None,
8894                signal_arguments: None,
8895                workflow_update_id: None,
8896                update_name: None,
8897                lease_owner: Some("rust-worker".to_string()),
8898            }
8899        }
8900
8901        let calls = Arc::new(AtomicUsize::new(0));
8902        let initial = worker(Arc::clone(&calls))
8903            .execute_workflow_task(task(Vec::new()))
8904            .expect("initial execution");
8905        assert_eq!(
8906            initial
8907                .iter()
8908                .map(|command| &command["type"])
8909                .collect::<Vec<_>>(),
8910            vec![
8911                "record_side_effect",
8912                "record_version_marker",
8913                "complete_workflow"
8914            ]
8915        );
8916        assert_eq!(calls.load(Ordering::SeqCst), 1);
8917
8918        let restarted = worker(Arc::clone(&calls));
8919        let replayed = restarted
8920            .execute_workflow_task(task(vec![
8921                history_event(
8922                    "SideEffectRecorded",
8923                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8924                ),
8925                history_event(
8926                    "VersionMarkerRecorded",
8927                    json!({
8928                        "sequence": 2,
8929                        "change_id": "cold-restart",
8930                        "version": 2,
8931                        "min_supported": 1,
8932                        "max_supported": 2,
8933                    }),
8934                ),
8935            ]))
8936            .expect("cold replay");
8937        assert_eq!(replayed.len(), 1);
8938        assert_eq!(replayed[0]["type"], "complete_workflow");
8939        assert_eq!(calls.load(Ordering::SeqCst), 1);
8940    }
8941
8942    #[test]
8943    fn side_effect_replay_rejects_changed_rust_value_type() {
8944        let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8945        let ctx = workflow_context(vec![history_event(
8946            "SideEffectRecorded",
8947            json!({"sequence": 1, "result": result}),
8948        )]);
8949        let error = ctx
8950            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8951            .expect_err("changed type must fail replay");
8952        assert!(matches!(
8953            error,
8954            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8955                if reason == "side_effect_type_mismatch"
8956        ));
8957    }
8958
8959    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8960        vec![
8961            history_event(
8962                "ActivityScheduled",
8963                json!({
8964                    "sequence": 1,
8965                    "activity_type": "flaky",
8966                    "activity_execution_id": "act-1",
8967                    "activity": {
8968                        "id": "act-1",
8969                        "sequence": 1,
8970                        "type": "flaky",
8971                        "queue": "critical-activities",
8972                        "execution_mode": null,
8973                        "retry_policy": {
8974                            "snapshot_version": 1,
8975                            "max_attempts": 3,
8976                            "backoff_seconds": [2, 4],
8977                            "start_to_close_timeout": 30,
8978                            "schedule_to_start_timeout": 5,
8979                            "schedule_to_close_timeout": 90,
8980                            "heartbeat_timeout": 10,
8981                            "non_retryable_error_types": ["PermanentError"]
8982                        }
8983                    }
8984                }),
8985            ),
8986            history_event(
8987                "ActivityStarted",
8988                json!({
8989                    "sequence": 1,
8990                    "activity_type": "flaky",
8991                    "activity_execution_id": "act-1",
8992                    "activity_attempt_id": "attempt-1",
8993                    "attempt_number": 1
8994                }),
8995            ),
8996            history_event(
8997                "ActivityRetryScheduled",
8998                json!({
8999                    "sequence": 1,
9000                    "activity_type": "flaky",
9001                    "activity_execution_id": "act-1",
9002                    "activity_attempt_id": "attempt-1",
9003                    "attempt_number": 1,
9004                    "retry_after_attempt": 1,
9005                    "retry_backoff_seconds": 2,
9006                    "failure_category": "activity",
9007                    "exception_type": "TransientError"
9008                }),
9009            ),
9010            history_event(
9011                "ActivityStarted",
9012                json!({
9013                    "sequence": 1,
9014                    "activity_type": "flaky",
9015                    "activity_execution_id": "act-1",
9016                    "activity_attempt_id": "attempt-2",
9017                    "attempt_number": 2
9018                }),
9019            ),
9020            history_event(
9021                "ActivityCompleted",
9022                json!({
9023                    "sequence": 1,
9024                    "activity_type": "flaky",
9025                    "activity_execution_id": "act-1",
9026                    "activity_attempt_id": "attempt-2",
9027                    "attempt_number": 2,
9028                    "payload_codec": "json",
9029                    "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
9030                }),
9031            ),
9032        ]
9033    }
9034
9035    fn retry_activity_options() -> ActivityOptions {
9036        ActivityOptions::new()
9037            .task_queue("critical-activities")
9038            .retry_policy(
9039                ActivityRetryPolicy::new(3)
9040                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9041                    .non_retryable_error_type("PermanentError"),
9042            )
9043            .start_to_close_timeout(Duration::from_secs(30))
9044            .schedule_to_start_timeout(Duration::from_secs(5))
9045            .schedule_to_close_timeout(Duration::from_secs(90))
9046            .heartbeat_timeout(Duration::from_secs(10))
9047    }
9048
9049    #[test]
9050    fn fixed_avro_value_round_trips_json_values() {
9051        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9052        let envelope = PayloadEnvelope::avro(&value).expect("encode");
9053        assert_eq!(envelope.codec, DEFAULT_CODEC);
9054        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9055    }
9056
9057    #[tokio::test]
9058    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9059        let client = Client::new("http://127.0.0.1:8080").expect("client");
9060        let mut worker = Worker::new(client, "rust-workers");
9061        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9062        worker
9063            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9064        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9065            Ok(input)
9066        });
9067        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9068            Ok(input)
9069        });
9070        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9071            Ok(AvroValue::Array(
9072                ctx.wait_signal_avro_value("changed").await?,
9073            ))
9074        });
9075
9076        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9077        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9078
9079        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9080        workflow.arguments = Some(envelope.clone());
9081        let commands = worker
9082            .execute_workflow_task(workflow)
9083            .expect("typed workflow task");
9084        assert_eq!(commands[0]["type"], "complete_workflow");
9085        assert_eq!(
9086            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9087                .expect("typed workflow result"),
9088            arguments
9089        );
9090
9091        let activity = ActivityTask {
9092            task_id: "activity-typed".to_string(),
9093            activity_attempt_id: Some("attempt-typed".to_string()),
9094            attempt_id: None,
9095            activity_type: "typed.activity".to_string(),
9096            payload_codec: DEFAULT_CODEC.to_string(),
9097            arguments: Some(envelope.clone()),
9098            attempt_number: 1,
9099            lease_owner: Some("rust-worker".to_string()),
9100        };
9101        assert_eq!(
9102            worker
9103                .execute_activity_task(activity)
9104                .await
9105                .expect("typed activity result"),
9106            arguments
9107        );
9108
9109        let query = QueryTask {
9110            query_task_id: "query-typed".to_string(),
9111            query_task_attempt: 1,
9112            lease_owner: Some("rust-worker".to_string()),
9113            workflow_id: Some("typed-1".to_string()),
9114            run_id: Some("run-typed".to_string()),
9115            workflow_type: "typed.echo".to_string(),
9116            query_name: "inspect".to_string(),
9117            payload_codec: DEFAULT_CODEC.to_string(),
9118            workflow_arguments: Some(
9119                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9120                    .expect("workflow input"),
9121            ),
9122            query_arguments: Some(envelope.clone()),
9123            history_events: Vec::new(),
9124            history_export: None,
9125            run_status: Some("running".to_string()),
9126        };
9127        assert_eq!(
9128            worker
9129                .execute_query_task(query)
9130                .await
9131                .expect("typed query result"),
9132            arguments
9133        );
9134
9135        let mut update = workflow_task(
9136            "typed.echo",
9137            vec![history_event(
9138                "UpdateAccepted",
9139                json!({
9140                    "update_id": "update-typed",
9141                    "update_name": "replace",
9142                    "arguments": envelope.clone(),
9143                }),
9144            )],
9145            DEFAULT_CODEC,
9146        );
9147        update.workflow_update_id = Some("update-typed".to_string());
9148        update.update_name = Some("replace".to_string());
9149        let commands = worker
9150            .execute_workflow_task(update)
9151            .expect("typed update task");
9152        assert_eq!(commands[0]["type"], "complete_update");
9153        assert_eq!(
9154            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9155                .expect("typed update result"),
9156            arguments
9157        );
9158
9159        let mut signal = workflow_task(
9160            "typed.signal",
9161            vec![history_event(
9162                "SignalReceived",
9163                json!({
9164                    "signal_id": "signal-typed",
9165                    "signal_name": "changed",
9166                    "arguments": envelope.clone(),
9167                }),
9168            )],
9169            DEFAULT_CODEC,
9170        );
9171        signal.workflow_signal_id = Some("signal-typed".to_string());
9172        signal.signal_name = Some("changed".to_string());
9173        signal.signal_arguments = Some(envelope);
9174        let commands = worker
9175            .execute_workflow_task(signal)
9176            .expect("typed signal resume");
9177        assert_eq!(
9178            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9179                .expect("typed signal result"),
9180            arguments
9181        );
9182    }
9183
9184    #[tokio::test]
9185    async fn typed_helpers_never_parse_json_inspection_projection() {
9186        let collision_values = projection_collision_probe();
9187        let expected = AvroValue::Array(collision_values.clone());
9188        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9189
9190        let activity_context = workflow_context_with_codec(
9191            vec![history_event(
9192                "ActivityCompleted",
9193                json!({
9194                    "sequence": 1,
9195                    "activity_type": "collision.activity",
9196                    "payload_codec": DEFAULT_CODEC,
9197                    "result": envelope.clone(),
9198                }),
9199            )],
9200            DEFAULT_CODEC,
9201        );
9202        assert_eq!(
9203            activity_context
9204                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9205                .await
9206                .expect("typed activity collision result"),
9207            expected
9208        );
9209
9210        let signal_context = workflow_context_with_codec(
9211            vec![
9212                history_event(
9213                    "SignalWaitOpened",
9214                    json!({"sequence": 1, "signal_name": "collision"}),
9215                ),
9216                history_event(
9217                    "SignalApplied",
9218                    json!({
9219                        "sequence": 1,
9220                        "signal_name": "collision",
9221                        "payload_codec": DEFAULT_CODEC,
9222                        "value": envelope.clone(),
9223                    }),
9224                ),
9225            ],
9226            DEFAULT_CODEC,
9227        );
9228        assert_eq!(
9229            signal_context
9230                .wait_signal_avro_value("collision")
9231                .await
9232                .expect("typed signal collision arguments"),
9233            collision_values
9234        );
9235
9236        let child_context = workflow_context_with_codec(
9237            vec![
9238                history_event(
9239                    "ChildWorkflowScheduled",
9240                    json!({
9241                        "sequence": 1,
9242                        "child_workflow_instance_id": "collision-child",
9243                        "child_workflow_run_id": "collision-run",
9244                        "child_workflow_type": "collision.child",
9245                    }),
9246                ),
9247                history_event(
9248                    "ChildRunCompleted",
9249                    json!({
9250                        "sequence": 1,
9251                        "child_workflow_instance_id": "collision-child",
9252                        "child_workflow_run_id": "collision-run",
9253                        "child_workflow_type": "collision.child",
9254                        "payload_codec": DEFAULT_CODEC,
9255                        "result": envelope,
9256                    }),
9257                ),
9258            ],
9259            DEFAULT_CODEC,
9260        );
9261        let child = child_context
9262            .start_child_workflow_avro_value(
9263                "collision.child",
9264                ChildWorkflowOptions::new("collision-workers"),
9265                AvroValue::Array(Vec::new()),
9266            )
9267            .await
9268            .expect("typed child collision result");
9269        assert_eq!(child.result, expected);
9270    }
9271
9272    #[tokio::test]
9273    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9274        let client = Client::new("http://127.0.0.1:8080").expect("client");
9275        let mut worker = Worker::new(client, "rust-workers");
9276        worker.register_replayed_workflow_avro_value(
9277            "typed.replayed",
9278            || (),
9279            |_ctx, input, _state| async move { Ok(input) },
9280        );
9281        worker.register_replayed_query_avro_value::<(), _, _>(
9282            "typed.replayed",
9283            "inspect",
9284            |ctx, _state, args| async move {
9285                let mut signals = ctx.signals_avro_value("collision");
9286                let signal = signals
9287                    .pop()
9288                    .map(AvroValue::Array)
9289                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9290                Ok(AvroValue::Array(vec![
9291                    ctx.workflow_input_avro_value().clone(),
9292                    signal,
9293                    args,
9294                ]))
9295            },
9296        );
9297        let arguments = AvroValue::Array(projection_collision_probe());
9298        let signal_arguments =
9299            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9300        let task = QueryTask {
9301            query_task_id: "query-typed-replay".to_string(),
9302            query_task_attempt: 1,
9303            lease_owner: Some("rust-worker".to_string()),
9304            workflow_id: Some("typed-replay".to_string()),
9305            run_id: Some("run-typed-replay".to_string()),
9306            workflow_type: "typed.replayed".to_string(),
9307            query_name: "inspect".to_string(),
9308            payload_codec: DEFAULT_CODEC.to_string(),
9309            workflow_arguments: Some(
9310                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9311            ),
9312            query_arguments: Some(
9313                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9314            ),
9315            history_events: vec![history_event(
9316                "SignalReceived",
9317                json!({
9318                    "signal_id": "collision-signal",
9319                    "signal_name": "collision",
9320                    "workflow_sequence": 1,
9321                    "payload_codec": DEFAULT_CODEC,
9322                    "arguments": signal_arguments,
9323                }),
9324            )],
9325            history_export: None,
9326            run_status: Some("completed".to_string()),
9327        };
9328
9329        assert_eq!(
9330            worker
9331                .execute_query_task(task)
9332                .await
9333                .expect("typed replay query"),
9334            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9335        );
9336    }
9337
9338    #[test]
9339    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9340        let value = BTreeMap::from([(1_i32, "integer key")]);
9341        let error = PayloadEnvelope::avro(&value)
9342            .expect_err("integer map keys must fail")
9343            .to_string();
9344
9345        assert!(error.contains("invalid_map_key"));
9346    }
9347
9348    #[test]
9349    fn json_codec_remains_plain_json() {
9350        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9351        let envelope = PayloadEnvelope::json(&value).expect("encode");
9352
9353        assert_eq!(envelope.codec, JSON_CODEC);
9354        assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9355        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9356    }
9357
9358    #[test]
9359    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9360        let envelope = PayloadEnvelope {
9361            codec: DEFAULT_CODEC.to_string(),
9362            blob: BASE64.encode([0x01]),
9363        };
9364
9365        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9366        assert!(error.to_string().contains("invalid_payload_framing"));
9367    }
9368
9369    #[test]
9370    fn workflow_context_schedules_activity_until_completion_is_in_history() {
9371        let ctx = WorkflowContext {
9372            state: Arc::new(Mutex::new(
9373                WorkflowState::new_with_identity(
9374                    Vec::new(),
9375                    Some("wf-parent".to_string()),
9376                    Some("run-parent".to_string()),
9377                    "rust-workers".to_string(),
9378                    DEFAULT_CODEC.to_string(),
9379                    None,
9380                )
9381                .expect("workflow state"),
9382            )),
9383        };
9384
9385        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9386        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9387        assert!(matches!(
9388            call.as_mut().poll(&mut task_context),
9389            Poll::Pending
9390        ));
9391
9392        let commands = ctx.take_commands().expect("commands");
9393        assert_eq!(commands[0]["type"], "schedule_activity");
9394        assert_eq!(commands[0]["activity_type"], "hello.activity");
9395    }
9396
9397    #[test]
9398    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9399        let ctx = workflow_context(Vec::new());
9400        let options = ActivityOptions::new()
9401            .task_queue("payments")
9402            .retry_policy(
9403                ActivityRetryPolicy::new(4)
9404                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9405                    .non_retryable_error_type("ValidationError"),
9406            )
9407            .start_to_close_timeout(Duration::from_secs(120))
9408            .schedule_to_start_timeout(Duration::from_secs(10))
9409            .schedule_to_close_timeout(Duration::from_secs(300))
9410            .heartbeat_timeout(Duration::from_secs(15));
9411        let mut call = Box::pin(ctx.activity_with_options(
9412            "charge-card",
9413            options,
9414            json!([{"order_id": "o-1"}]),
9415        ));
9416        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9417
9418        assert!(matches!(
9419            call.as_mut().poll(&mut task_context),
9420            Poll::Pending
9421        ));
9422        assert!(matches!(
9423            call.as_mut().poll(&mut task_context),
9424            Poll::Pending
9425        ));
9426
9427        let commands = ctx.take_commands().expect("activity command");
9428        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9429        assert_eq!(commands[0]["queue"], "payments");
9430        assert_eq!(
9431            commands[0]["retry_policy"],
9432            json!({
9433                "max_attempts": 4,
9434                "backoff_seconds": [1, 3, 9],
9435                "non_retryable_error_types": ["ValidationError"],
9436            })
9437        );
9438        assert_eq!(commands[0]["start_to_close_timeout"], 120);
9439        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9440        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9441        assert_eq!(commands[0]["heartbeat_timeout"], 15);
9442    }
9443
9444    #[test]
9445    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9446        let ctx = workflow_context(Vec::new());
9447        let options = ActivityOptions::new().retry_policy(
9448            ActivityRetryPolicy::new(3)
9449                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9450        );
9451        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9452        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9453
9454        assert!(matches!(
9455            call.as_mut().poll(&mut task_context),
9456            Poll::Pending
9457        ));
9458        assert_eq!(
9459            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9460            json!([1, 2])
9461        );
9462    }
9463
9464    #[test]
9465    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9466        let cases = [
9467            (
9468                ActivityOptions::new().task_queue("  "),
9469                ActivityOptionsErrorKind::EmptyTaskQueue,
9470            ),
9471            (
9472                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9473                ActivityOptionsErrorKind::EmptyRetryPolicy,
9474            ),
9475            (
9476                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9477                ActivityOptionsErrorKind::InvalidMaxAttempts,
9478            ),
9479            (
9480                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9481                    max_attempts: None,
9482                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9483                    non_retryable_error_types: Vec::new(),
9484                }),
9485                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9486            ),
9487            (
9488                ActivityOptions::new().retry_policy(
9489                    ActivityRetryPolicy::new(2)
9490                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9491                ),
9492                ActivityOptionsErrorKind::TooManyBackoffIntervals,
9493            ),
9494            (
9495                ActivityOptions::new().retry_policy(
9496                    ActivityRetryPolicy::new(2).exponential_backoff(
9497                        Duration::from_secs(1),
9498                        0,
9499                        None,
9500                    ),
9501                ),
9502                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9503            ),
9504            (
9505                ActivityOptions::new()
9506                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
9507                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9508            ),
9509            (
9510                ActivityOptions::new().retry_policy(
9511                    ActivityRetryPolicy::new(10_002).exponential_backoff(
9512                        Duration::from_secs(1),
9513                        1,
9514                        None,
9515                    ),
9516                ),
9517                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9518            ),
9519            (
9520                ActivityOptions::new().retry_policy(
9521                    ActivityRetryPolicy::new(2)
9522                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9523                ),
9524                ActivityOptionsErrorKind::BackoffOverflow,
9525            ),
9526        ];
9527
9528        for (options, expected_kind) in cases {
9529            let ctx = workflow_context(Vec::new());
9530            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9531            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9532            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9533                call.as_mut().poll(&mut task_context)
9534            else {
9535                panic!("expected typed activity validation error");
9536            };
9537            assert_eq!(error.kind, expected_kind);
9538            assert!(ctx.take_commands().expect("commands").is_empty());
9539        }
9540    }
9541
9542    #[test]
9543    fn activity_options_validate_positive_and_ordered_timeouts() {
9544        let zero_timeout_cases = [
9545            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9546            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9547            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9548            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9549        ];
9550        for options in zero_timeout_cases {
9551            assert_eq!(
9552                options.validate().expect_err("zero timeout").kind,
9553                ActivityOptionsErrorKind::TimeoutNotPositive
9554            );
9555        }
9556
9557        let ordering_cases = [
9558            ActivityOptions::new()
9559                .heartbeat_timeout(Duration::from_secs(11))
9560                .start_to_close_timeout(Duration::from_secs(10)),
9561            ActivityOptions::new()
9562                .start_to_close_timeout(Duration::from_secs(31))
9563                .schedule_to_close_timeout(Duration::from_secs(30)),
9564            ActivityOptions::new()
9565                .schedule_to_start_timeout(Duration::from_secs(31))
9566                .schedule_to_close_timeout(Duration::from_secs(30)),
9567        ];
9568        for options in ordering_cases {
9569            assert_eq!(
9570                options.validate().expect_err("timeout order").kind,
9571                ActivityOptionsErrorKind::TimeoutOrder
9572            );
9573        }
9574
9575        assert_eq!(
9576            ActivityOptions::new()
9577                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9578                .validate()
9579                .expect_err("protocol integer overflow")
9580                .kind,
9581            ActivityOptionsErrorKind::TimeoutOverflow
9582        );
9583    }
9584
9585    #[test]
9586    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9587        let ctx = workflow_context(completed_retry_activity_history());
9588        let mut call =
9589            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9590        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9591
9592        assert!(matches!(
9593            call.as_mut().poll(&mut task_context),
9594            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9595        ));
9596        assert!(ctx.take_commands().expect("commands").is_empty());
9597        ctx.ensure_history_consumed().expect("history consumed");
9598    }
9599
9600    #[test]
9601    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9602        let mut options = retry_activity_options();
9603        options
9604            .retry_policy
9605            .as_mut()
9606            .expect("retry policy")
9607            .non_retryable_error_types
9608            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9609
9610        let new_ctx = workflow_context(Vec::new());
9611        let mut new_call =
9612            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9613        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9614        assert!(matches!(
9615            new_call.as_mut().poll(&mut task_context),
9616            Poll::Pending
9617        ));
9618        let commands = new_ctx.take_commands().expect("commands");
9619        assert_eq!(commands.len(), 1);
9620        assert_eq!(
9621            commands[0]["retry_policy"]["non_retryable_error_types"],
9622            json!(["PermanentError"])
9623        );
9624
9625        let replay_ctx = workflow_context(completed_retry_activity_history());
9626        let mut replay_call =
9627            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9628        assert!(matches!(
9629            replay_call.as_mut().poll(&mut task_context),
9630            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9631        ));
9632        assert!(replay_ctx.take_commands().expect("commands").is_empty());
9633        replay_ctx
9634            .ensure_history_consumed()
9635            .expect("history consumed");
9636    }
9637
9638    #[test]
9639    fn replayed_intermediate_retry_remains_pending_across_restarts() {
9640        let history = completed_retry_activity_history()
9641            .into_iter()
9642            .take(3)
9643            .collect::<Vec<_>>();
9644
9645        for _restart in 0..2 {
9646            let ctx = workflow_context(history.clone());
9647            let mut call =
9648                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9649            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9650            assert!(matches!(
9651                call.as_mut().poll(&mut task_context),
9652                Poll::Pending
9653            ));
9654            assert!(ctx.take_commands().expect("commands").is_empty());
9655        }
9656    }
9657
9658    #[test]
9659    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9660        let mut changed_queue = retry_activity_options();
9661        changed_queue.task_queue = Some("different-queue".to_string());
9662
9663        let mut changed_max_attempts = retry_activity_options();
9664        let retry_policy = changed_max_attempts
9665            .retry_policy
9666            .as_mut()
9667            .expect("retry policy");
9668        retry_policy.max_attempts = Some(4);
9669
9670        let mut changed_backoff = retry_activity_options();
9671        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9672        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9673            Duration::from_secs(3),
9674            Duration::from_secs(4),
9675        ]));
9676
9677        let mut changed_non_retryable_types = retry_activity_options();
9678        let retry_policy = changed_non_retryable_types
9679            .retry_policy
9680            .as_mut()
9681            .expect("retry policy");
9682        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9683
9684        let mut changed_start_to_close = retry_activity_options();
9685        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9686        let mut changed_schedule_to_start = retry_activity_options();
9687        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9688        let mut changed_schedule_to_close = retry_activity_options();
9689        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9690        let mut changed_heartbeat = retry_activity_options();
9691        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9692
9693        let cases = [
9694            (changed_queue, "activity_task_queue_mismatch"),
9695            (changed_max_attempts, "activity_retry_policy_mismatch"),
9696            (changed_backoff, "activity_retry_policy_mismatch"),
9697            (
9698                changed_non_retryable_types,
9699                "activity_retry_policy_mismatch",
9700            ),
9701            (changed_start_to_close, "activity_retry_policy_mismatch"),
9702            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9703            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9704            (changed_heartbeat, "activity_retry_policy_mismatch"),
9705        ];
9706
9707        for (options, expected_reason) in cases {
9708            let ctx = workflow_context(completed_retry_activity_history());
9709            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9710            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9711            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9712                call.as_mut().poll(&mut task_context)
9713            else {
9714                panic!("changed activity options must fail replay");
9715            };
9716            assert_eq!(failure.reason, expected_reason);
9717            assert_eq!(failure.sequence, Some(1));
9718            assert!(ctx.take_commands().expect("commands").is_empty());
9719        }
9720    }
9721
9722    #[test]
9723    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9724        let cases = [
9725            (
9726                "execution_mode",
9727                json!("local"),
9728                "activity_execution_mode_mismatch",
9729            ),
9730            (
9731                "snapshot_version",
9732                json!(2),
9733                "activity_retry_policy_mismatch",
9734            ),
9735        ];
9736
9737        for (field, value, expected_reason) in cases {
9738            let mut history = completed_retry_activity_history();
9739            let activity = history[0].payload["activity"]
9740                .as_object_mut()
9741                .expect("activity snapshot");
9742            if field == "execution_mode" {
9743                activity.insert(field.to_string(), value);
9744            } else {
9745                activity["retry_policy"]
9746                    .as_object_mut()
9747                    .expect("retry snapshot")
9748                    .insert(field.to_string(), value);
9749            }
9750
9751            let ctx = workflow_context(history);
9752            let mut call =
9753                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9754            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9755            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9756                call.as_mut().poll(&mut task_context)
9757            else {
9758                panic!("changed {field} must fail replay");
9759            };
9760            assert_eq!(failure.reason, expected_reason);
9761            assert_eq!(failure.sequence, Some(1));
9762            assert!(ctx.take_commands().expect("commands").is_empty());
9763        }
9764    }
9765
9766    #[test]
9767    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9768        let mut history = completed_retry_activity_history();
9769        let activity = history[0].payload["activity"]
9770            .as_object_mut()
9771            .expect("activity snapshot");
9772        activity.remove("execution_mode");
9773        activity.remove("retry_policy");
9774
9775        let mut current = retry_activity_options();
9776        current.start_to_close_timeout = Some(Duration::from_secs(45));
9777        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9778        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9779        current.heartbeat_timeout = Some(Duration::from_secs(12));
9780
9781        let ctx = workflow_context(history);
9782        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9783        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9784        assert!(matches!(
9785            call.as_mut().poll(&mut task_context),
9786            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9787        ));
9788        assert!(ctx.take_commands().expect("commands").is_empty());
9789        ctx.ensure_history_consumed().expect("history consumed");
9790    }
9791
9792    #[test]
9793    fn terminal_activity_failed_after_start_returns_typed_failure() {
9794        let history = vec![
9795            history_event(
9796                "ActivityScheduled",
9797                json!({
9798                    "sequence": 1,
9799                    "activity_type": "flaky",
9800                    "activity_execution_id": "act-terminal",
9801                    "activity": {
9802                        "id": "act-terminal",
9803                        "sequence": 1,
9804                        "type": "flaky",
9805                        "queue": "critical-activities",
9806                        "retry_policy": {
9807                            "snapshot_version": 1,
9808                            "max_attempts": 3,
9809                            "backoff_seconds": [2, 4],
9810                            "non_retryable_error_types": ["PermanentError"]
9811                        }
9812                    }
9813                }),
9814            ),
9815            history_event(
9816                "ActivityStarted",
9817                json!({
9818                    "sequence": 1,
9819                    "activity_type": "flaky",
9820                    "activity_execution_id": "act-terminal",
9821                    "activity_attempt_id": "attempt-1",
9822                    "attempt_number": 1
9823                }),
9824            ),
9825            history_event(
9826                "ActivityFailed",
9827                json!({
9828                    "sequence": 1,
9829                    "activity_type": "flaky",
9830                    "activity_execution_id": "act-terminal",
9831                    "activity_attempt_id": "attempt-1",
9832                    "attempt_number": 1,
9833                    "failure_id": "failure-terminal",
9834                    "failure_category": "activity",
9835                    "exception_type": "PermanentError",
9836                    "message": "cannot retry",
9837                    "non_retryable": true
9838                }),
9839            ),
9840        ];
9841        let ctx = workflow_context(history);
9842        let mut call =
9843            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9844        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9845
9846        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9847            call.as_mut().poll(&mut task_context)
9848        else {
9849            panic!("terminal ActivityFailed must settle the activity future");
9850        };
9851        assert_eq!(failure.kind, ActivityFailureKind::Failed);
9852        assert_eq!(
9853            failure.activity_execution_id.as_deref(),
9854            Some("act-terminal")
9855        );
9856        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9857        assert!(failure.non_retryable);
9858        assert!(ctx.take_commands().expect("commands").is_empty());
9859        ctx.ensure_history_consumed().expect("history consumed");
9860    }
9861
9862    #[test]
9863    fn activity_terminal_events_return_machine_readable_failures() {
9864        let cases = [
9865            (
9866                "ActivityFailed",
9867                json!({
9868                    "sequence": 1,
9869                    "activity_type": "charge-card",
9870                    "activity_execution_id": "act-1",
9871                    "activity_attempt_id": "attempt-2",
9872                    "attempt_number": 2,
9873                    "failure_id": "failure-1",
9874                    "failure_category": "activity",
9875                    "exception_type": "PaymentDeclined",
9876                    "exception_class": "payments.PaymentDeclined",
9877                    "message": "card declined",
9878                    "non_retryable": true
9879                }),
9880                ActivityFailureKind::Failed,
9881                "activity",
9882            ),
9883            (
9884                "ActivityCancelled",
9885                json!({
9886                    "sequence": 1,
9887                    "activity_type": "charge-card",
9888                    "activity_execution_id": "act-1",
9889                    "activity_attempt_id": "attempt-1"
9890                }),
9891                ActivityFailureKind::Cancelled,
9892                "cancelled",
9893            ),
9894        ];
9895
9896        for (event_type, payload, expected_kind, expected_reason) in cases {
9897            let ctx = workflow_context(vec![history_event(event_type, payload)]);
9898            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9899            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9900            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9901                call.as_mut().poll(&mut task_context)
9902            else {
9903                panic!("expected terminal activity failure");
9904            };
9905            assert_eq!(failure.kind, expected_kind);
9906            assert_eq!(failure.reason, expected_reason);
9907            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9908            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9909        }
9910    }
9911
9912    #[test]
9913    fn every_activity_timeout_class_is_typed() {
9914        for timeout_kind in [
9915            "start_to_close",
9916            "schedule_to_start",
9917            "schedule_to_close",
9918            "heartbeat",
9919        ] {
9920            let ctx = workflow_context(vec![history_event(
9921                "ActivityTimedOut",
9922                json!({
9923                    "sequence": 1,
9924                    "activity_type": "slow",
9925                    "activity_execution_id": "act-timeout",
9926                    "activity_attempt_id": "attempt-timeout",
9927                    "failure_category": "timeout",
9928                    "timeout_kind": timeout_kind,
9929                    "message": "deadline expired"
9930                }),
9931            )]);
9932            let mut call = Box::pin(ctx.activity("slow", json!([])));
9933            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9934            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9935                call.as_mut().poll(&mut task_context)
9936            else {
9937                panic!("expected timeout failure");
9938            };
9939            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9940            assert_eq!(failure.reason, timeout_kind);
9941            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9942            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9943        }
9944    }
9945
9946    #[test]
9947    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9948        let ctx = workflow_context(Vec::new());
9949        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9950        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9951
9952        assert!(matches!(
9953            sleep.as_mut().poll(&mut task_context),
9954            Poll::Pending
9955        ));
9956        assert!(matches!(
9957            sleep.as_mut().poll(&mut task_context),
9958            Poll::Pending
9959        ));
9960
9961        let commands = ctx.take_commands().expect("timer command");
9962        assert_eq!(
9963            commands,
9964            vec![json!({
9965                "type": "start_timer",
9966                "delay_seconds": 2,
9967            })]
9968        );
9969    }
9970
9971    #[test]
9972    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9973        let history = vec![
9974            history_event(
9975                "TimerScheduled",
9976                json!({
9977                    "sequence": 1,
9978                    "timer_id": "timer-1",
9979                    "delay_seconds": 5,
9980                    "fire_at": "2026-07-11T12:00:05Z",
9981                }),
9982            ),
9983            history_event(
9984                "TimerFired",
9985                json!({
9986                    "sequence": 1,
9987                    "timer_id": "timer-1",
9988                    "delay_seconds": 5,
9989                    "fire_at": "2026-07-11T12:00:05Z",
9990                    "fired_at": "2026-07-11T12:00:05Z",
9991                }),
9992            ),
9993        ];
9994
9995        for _restart in 0..2 {
9996            let ctx = workflow_context(history.clone());
9997            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
9998            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9999            assert!(matches!(
10000                sleep.as_mut().poll(&mut task_context),
10001                Poll::Ready(Ok(()))
10002            ));
10003            assert!(ctx.take_commands().expect("commands").is_empty());
10004            ctx.ensure_history_consumed().expect("history consumed");
10005        }
10006    }
10007
10008    #[test]
10009    fn workflow_sleep_rejects_changed_delay_during_replay() {
10010        let ctx = workflow_context(vec![
10011            history_event(
10012                "TimerScheduled",
10013                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10014            ),
10015            history_event(
10016                "TimerFired",
10017                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10018            ),
10019        ]);
10020        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10021        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10022
10023        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10024            sleep.as_mut().poll(&mut task_context)
10025        else {
10026            panic!("changed timer delay must be rejected");
10027        };
10028        assert_eq!(failure.reason, "timer_delay_mismatch");
10029        assert_eq!(failure.sequence, Some(1));
10030    }
10031
10032    #[test]
10033    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10034        let lone_fire = WorkflowState::new(
10035            vec![history_event(
10036                "TimerFired",
10037                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10038            )],
10039            "rust-workers".to_string(),
10040            JSON_CODEC.to_string(),
10041            None,
10042        )
10043        .expect_err("TimerFired requires TimerScheduled");
10044        assert!(matches!(
10045            lone_fire,
10046            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10047                if reason == "timer_schedule_missing_or_duplicate"
10048        ));
10049
10050        let wrong_identity = WorkflowState::new(
10051            vec![
10052                history_event(
10053                    "TimerScheduled",
10054                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10055                ),
10056                history_event(
10057                    "TimerFired",
10058                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10059                ),
10060            ],
10061            "rust-workers".to_string(),
10062            JSON_CODEC.to_string(),
10063            None,
10064        )
10065        .expect_err("fire must match scheduled timer identity");
10066        assert!(matches!(
10067            wrong_identity,
10068            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10069                if reason == "timer_identity_mismatch"
10070        ));
10071
10072        let duplicate_fire = WorkflowState::new(
10073            vec![
10074                history_event(
10075                    "TimerScheduled",
10076                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10077                ),
10078                history_event(
10079                    "TimerFired",
10080                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10081                ),
10082                history_event(
10083                    "TimerFired",
10084                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10085                ),
10086            ],
10087            "rust-workers".to_string(),
10088            JSON_CODEC.to_string(),
10089            None,
10090        )
10091        .expect_err("a durable timer cannot fire twice");
10092        assert!(matches!(
10093            duplicate_fire,
10094            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10095                if reason == "duplicate_timer_fire"
10096        ));
10097
10098        let wrong_fired_delay = WorkflowState::new(
10099            vec![
10100                history_event(
10101                    "TimerScheduled",
10102                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10103                ),
10104                history_event(
10105                    "TimerFired",
10106                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10107                ),
10108            ],
10109            "rust-workers".to_string(),
10110            JSON_CODEC.to_string(),
10111            None,
10112        )
10113        .expect_err("timer schedule and fire delays must agree");
10114        assert!(matches!(
10115            wrong_fired_delay,
10116            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10117                if reason == "timer_history_delay_mismatch"
10118        ));
10119    }
10120
10121    #[test]
10122    fn replay_rejects_activity_moved_before_recorded_timer() {
10123        let ctx = workflow_context(vec![
10124            history_event(
10125                "TimerScheduled",
10126                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10127            ),
10128            history_event(
10129                "TimerFired",
10130                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10131            ),
10132            history_event(
10133                "ActivityCompleted",
10134                json!({
10135                    "sequence": 2,
10136                    "activity_type": "after-timer",
10137                    "payload_codec": "json",
10138                    "result": {"codec": "json", "blob": "\"done\""},
10139                }),
10140            ),
10141        ]);
10142        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10143        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10144
10145        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10146            activity.as_mut().poll(&mut task_context)
10147        else {
10148            panic!("reordered durable command must be rejected");
10149        };
10150        assert_eq!(failure.reason, "recorded_command_mismatch");
10151        assert_eq!(failure.sequence, Some(1));
10152        assert_eq!(failure.expected.as_deref(), Some("timer"));
10153        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10154    }
10155
10156    #[test]
10157    fn workflow_context_emits_a_typed_named_signal_wait() {
10158        let ctx = workflow_context(Vec::new());
10159        let mut signal = Box::pin(ctx.wait_signal("finish"));
10160        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10161
10162        assert!(matches!(
10163            signal.as_mut().poll(&mut task_context),
10164            Poll::Pending
10165        ));
10166        assert_eq!(
10167            ctx.take_commands().expect("signal-wait command"),
10168            vec![json!({
10169                "type": "open_signal_wait",
10170                "signal_name": "finish",
10171            })]
10172        );
10173    }
10174
10175    #[test]
10176    fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10177        let ctx = workflow_context(vec![
10178            history_event(
10179                "ConditionWaitOpened",
10180                json!({"sequence": 1, "condition_key": "signal:finish"}),
10181            ),
10182            history_event(
10183                "ConditionWaitSatisfied",
10184                json!({"sequence": 1, "condition_key": "signal:finish"}),
10185            ),
10186            history_event(
10187                "SignalReceived",
10188                json!({"signal_name": "finish", "arguments": []}),
10189            ),
10190        ]);
10191        let mut signal = Box::pin(ctx.wait_signal("finish"));
10192        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10193
10194        assert!(matches!(
10195            signal.as_mut().poll(&mut task_context),
10196            Poll::Pending
10197        ));
10198        assert_eq!(
10199            ctx.take_commands().expect("typed signal-wait command"),
10200            vec![json!({
10201                "type": "open_signal_wait",
10202                "signal_name": "finish",
10203            })]
10204        );
10205    }
10206
10207    #[test]
10208    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10209        let signal_then_timer = vec![
10210            history_event(
10211                "SignalWaitOpened",
10212                json!({"sequence": 1, "signal_name": "go"}),
10213            ),
10214            history_event(
10215                "SignalApplied",
10216                json!({
10217                    "sequence": 1,
10218                    "signal_name": "go",
10219                    "value": {"codec": "json", "blob": "[\"now\"]"},
10220                }),
10221            ),
10222            history_event(
10223                "TimerScheduled",
10224                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10225            ),
10226            history_event(
10227                "TimerFired",
10228                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10229            ),
10230        ];
10231
10232        let ctx = workflow_context(signal_then_timer.clone());
10233        let mut signal = Box::pin(ctx.wait_signal("go"));
10234        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10235        assert!(matches!(
10236            signal.as_mut().poll(&mut task_context),
10237            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10238        ));
10239        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10240        assert!(matches!(
10241            timer.as_mut().poll(&mut task_context),
10242            Poll::Ready(Ok(()))
10243        ));
10244        ctx.ensure_history_consumed()
10245            .expect("signal and timer history consumed in order");
10246
10247        let reordered = workflow_context(signal_then_timer);
10248        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10249        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10250            timer_first.as_mut().poll(&mut task_context)
10251        else {
10252            panic!("timer cannot consume signal-wait-first history");
10253        };
10254        assert_eq!(failure.reason, "recorded_command_mismatch");
10255        assert_eq!(failure.sequence, Some(1));
10256        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10257
10258        let timer_then_signal = vec![
10259            history_event(
10260                "TimerScheduled",
10261                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10262            ),
10263            history_event(
10264                "TimerFired",
10265                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10266            ),
10267            history_event(
10268                "SignalWaitOpened",
10269                json!({"sequence": 2, "signal_name": "go"}),
10270            ),
10271            history_event(
10272                "SignalApplied",
10273                json!({
10274                    "sequence": 2,
10275                    "signal_name": "go",
10276                    "value": {"codec": "json", "blob": "[]"},
10277                }),
10278            ),
10279        ];
10280        let reordered = workflow_context(timer_then_signal);
10281        let mut signal_first = Box::pin(reordered.wait_signal("go"));
10282        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10283            signal_first.as_mut().poll(&mut task_context)
10284        else {
10285            panic!("signal wait cannot consume timer-first history");
10286        };
10287        assert_eq!(failure.reason, "recorded_command_mismatch");
10288        assert_eq!(failure.sequence, Some(1));
10289        assert_eq!(failure.expected.as_deref(), Some("timer"));
10290    }
10291
10292    #[test]
10293    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10294        let duplicate_timer = WorkflowState::new(
10295            vec![
10296                history_event(
10297                    "TimerScheduled",
10298                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10299                ),
10300                history_event(
10301                    "TimerScheduled",
10302                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10303                ),
10304            ],
10305            "rust-workers".to_string(),
10306            JSON_CODEC.to_string(),
10307            None,
10308        )
10309        .expect_err("one workflow sequence cannot schedule two timers");
10310        assert!(matches!(
10311            duplicate_timer,
10312            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10313                if reason == "timer_schedule_missing_or_duplicate"
10314        ));
10315
10316        let colliding_kinds = WorkflowState::new(
10317            vec![
10318                history_event(
10319                    "TimerScheduled",
10320                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10321                ),
10322                history_event(
10323                    "ActivityCompleted",
10324                    json!({"sequence": 1, "activity_type": "same-sequence"}),
10325                ),
10326            ],
10327            "rust-workers".to_string(),
10328            JSON_CODEC.to_string(),
10329            None,
10330        )
10331        .expect_err("one workflow sequence cannot identify two command kinds");
10332        assert!(matches!(
10333            colliding_kinds,
10334            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10335                if reason == "durable_command_sequence_collision"
10336        ));
10337
10338        let duplicate_signal_wait = WorkflowState::new(
10339            vec![
10340                history_event(
10341                    "SignalWaitOpened",
10342                    json!({"sequence": 1, "signal_name": "go"}),
10343                ),
10344                history_event(
10345                    "SignalWaitOpened",
10346                    json!({"sequence": 1, "signal_name": "go"}),
10347                ),
10348            ],
10349            "rust-workers".to_string(),
10350            JSON_CODEC.to_string(),
10351            None,
10352        )
10353        .expect_err("one workflow sequence cannot open two signal waits");
10354        assert!(matches!(
10355            duplicate_signal_wait,
10356            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10357                if reason == "signal_wait_open_missing_or_duplicate"
10358        ));
10359    }
10360
10361    #[test]
10362    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10363        let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10364            .expect("side-effect result");
10365        let ctx = workflow_context(vec![history_event(
10366            "SideEffectRecorded",
10367            json!({"sequence": 99, "result": result}),
10368        )]);
10369
10370        let replayed: Value = ctx
10371            .side_effect(|| panic!("recorded side effect must not run"))
10372            .expect("positive global workflow sequence is valid");
10373        assert_eq!(replayed, json!({"captured": true}));
10374        ctx.ensure_history_consumed().expect("history consumed");
10375    }
10376
10377    #[test]
10378    fn workflow_history_rejects_zero_and_descending_command_sequences() {
10379        let result =
10380            encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10381        let zero = WorkflowState::new(
10382            vec![history_event(
10383                "SideEffectRecorded",
10384                json!({"sequence": 0, "result": result.clone()}),
10385            )],
10386            "rust-workers".to_string(),
10387            JSON_CODEC.to_string(),
10388            None,
10389        )
10390        .expect_err("durable command sequences must be positive");
10391        assert!(matches!(
10392            zero,
10393            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10394                if reason == "durable_command_sequence_invalid"
10395        ));
10396
10397        let descending = WorkflowState::new(
10398            vec![
10399                history_event(
10400                    "SideEffectRecorded",
10401                    json!({"sequence": 3, "result": result}),
10402                ),
10403                history_event(
10404                    "VersionMarkerRecorded",
10405                    json!({
10406                        "sequence": 2,
10407                        "change_id": "descending-marker",
10408                        "version": 1,
10409                        "min_supported": 1,
10410                        "max_supported": 1,
10411                    }),
10412                ),
10413            ],
10414            "rust-workers".to_string(),
10415            JSON_CODEC.to_string(),
10416            None,
10417        )
10418        .expect_err("new durable commands must remain strictly ordered");
10419        let Error::NonDeterministicReplay(failure) = descending else {
10420            panic!("expected typed replay failure");
10421        };
10422        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10423        assert_eq!(failure.sequence, Some(2));
10424        assert_eq!(
10425            failure.expected.as_deref(),
10426            Some("workflow sequence greater than 3")
10427        );
10428        assert_eq!(failure.actual.as_deref(), Some("2"));
10429    }
10430
10431    #[test]
10432    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10433        fn worker() -> Worker {
10434            let client = Client::new("http://127.0.0.1:8080").expect("client");
10435            let mut worker = Worker::new(client, "rust-workers");
10436            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10437                ctx.wait_signal("finish").await?;
10438                let marker: String =
10439                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10440                assert_eq!(marker, "after-finish");
10441                Ok(json!("finished"))
10442            });
10443            worker
10444        }
10445
10446        let marker =
10447            encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10448        let task = workflow_task(
10449            "rust.finish-after-gaps",
10450            vec![
10451                history_event(
10452                    "SignalWaitOpened",
10453                    json!({"sequence": 1, "signal_name": "finish"}),
10454                ),
10455                history_event(
10456                    "SignalReceived",
10457                    json!({
10458                        "signal_id": "increment-3",
10459                        "signal_name": "increment",
10460                        "workflow_sequence": 2,
10461                        "payload_codec": "json",
10462                        "arguments": {"codec": "json", "blob": "[3]"},
10463                    }),
10464                ),
10465                history_event(
10466                    "SignalReceived",
10467                    json!({
10468                        "signal_id": "increment-5",
10469                        "signal_name": "increment",
10470                        "workflow_sequence": 3,
10471                        "payload_codec": "json",
10472                        "arguments": {"codec": "json", "blob": "[5]"},
10473                    }),
10474                ),
10475                history_event(
10476                    "SignalReceived",
10477                    json!({
10478                        "signal_id": "finish",
10479                        "signal_name": "finish",
10480                        "workflow_sequence": 4,
10481                        "payload_codec": "json",
10482                        "arguments": {"codec": "json", "blob": "[]"},
10483                    }),
10484                ),
10485                history_event(
10486                    "SignalApplied",
10487                    json!({
10488                        "sequence": 1,
10489                        "signal_id": "finish",
10490                        "signal_name": "finish",
10491                        "payload_codec": "json",
10492                        "value": {"codec": "json", "blob": "[]"},
10493                    }),
10494                ),
10495                history_event(
10496                    "SideEffectRecorded",
10497                    json!({"sequence": 5, "result": marker}),
10498                ),
10499            ],
10500            JSON_CODEC,
10501        );
10502
10503        for _original_or_cold_worker in 0..2 {
10504            let commands = worker()
10505                .execute_workflow_task(task.clone())
10506                .expect("signal gaps preserve deterministic replay");
10507            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10508            assert_eq!(commands[0]["type"], "complete_workflow");
10509            assert_eq!(
10510                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10511                json!("finished")
10512            );
10513        }
10514    }
10515
10516    #[test]
10517    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10518        let ctx = workflow_context(Vec::new());
10519        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10520        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10521        assert!(matches!(
10522            sleep.as_mut().poll(&mut task_context),
10523            Poll::Ready(Err(Error::TimerDurationOverflow))
10524        ));
10525        assert!(ctx.take_commands().expect("commands").is_empty());
10526    }
10527
10528    #[test]
10529    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10530        let client = Client::new("http://127.0.0.1:8080").expect("client");
10531        let mut worker = Worker::new(client, "rust-workers");
10532        worker.register_workflow("rust.timer", |ctx, _input| async move {
10533            ctx.sleep(Duration::from_secs(5)).await?;
10534            ctx.activity("after-timer", json!([])).await
10535        });
10536
10537        let task = |history_events| WorkflowTask {
10538            task_id: "wft-rust-timer-1".to_string(),
10539            workflow_id: Some("wf-rust-timer".to_string()),
10540            run_id: Some("run-rust-timer".to_string()),
10541            workflow_type: "rust.timer".to_string(),
10542            payload_codec: JSON_CODEC.to_string(),
10543            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10544            history_events,
10545            total_history_events: None,
10546            history_size_bytes: None,
10547            continue_as_new_recommended: None,
10548            history_budget_pressure: None,
10549            next_history_page_token: None,
10550            workflow_task_attempt: 1,
10551            workflow_signal_id: None,
10552            signal_name: None,
10553            signal_arguments: None,
10554            workflow_update_id: None,
10555            update_name: None,
10556            lease_owner: Some("rust-worker".to_string()),
10557        };
10558
10559        let initial = worker
10560            .execute_workflow_task(task(Vec::new()))
10561            .expect("initial timer task");
10562        assert_eq!(
10563            initial,
10564            vec![json!({"type": "start_timer", "delay_seconds": 5})]
10565        );
10566
10567        let replayed = worker
10568            .execute_workflow_task(task(vec![
10569                history_event(
10570                    "TimerScheduled",
10571                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10572                ),
10573                history_event(
10574                    "TimerFired",
10575                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10576                ),
10577                history_event(
10578                    "ActivityCompleted",
10579                    json!({
10580                        "sequence": 2,
10581                        "activity_type": "after-timer",
10582                        "payload_codec": "json",
10583                        "result": {"codec": "json", "blob": "\"done\""},
10584                    }),
10585                ),
10586            ]))
10587            .expect("replayed workflow task");
10588        assert_eq!(replayed.len(), 1);
10589        assert_eq!(replayed[0]["type"], "complete_workflow");
10590        assert_eq!(
10591            decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10592            json!("done")
10593        );
10594    }
10595
10596    #[test]
10597    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10598        let client = Client::new("http://127.0.0.1:8080").expect("client");
10599        let mut worker = Worker::new(client, "rust-workers");
10600        worker.register_workflow("rust.continue", |ctx, _input| async move {
10601            ctx.continue_as_new_with_options(
10602                ContinueAsNewOptions::new()
10603                    .workflow_type("rust.next")
10604                    .task_queue("next-workers"),
10605                json!([2, {"cursor": "next"}]),
10606            )
10607        });
10608
10609        let commands = worker
10610            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10611            .expect("continue-as-new command");
10612
10613        assert_eq!(commands.len(), 1);
10614        assert_eq!(commands[0]["type"], "continue_as_new");
10615        assert_eq!(commands[0]["workflow_type"], "rust.next");
10616        assert_eq!(commands[0]["queue"], "next-workers");
10617        assert_eq!(
10618            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10619                .expect("continue-as-new arguments"),
10620            json!([2, {"cursor": "next"}])
10621        );
10622    }
10623
10624    #[test]
10625    fn continue_as_new_preserves_typed_arguments() {
10626        let client = Client::new("http://127.0.0.1:8080").expect("client");
10627        let mut worker = Worker::new(client, "rust-workers");
10628        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10629            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10630            unreachable!("continue-as-new returns a control-flow error")
10631        });
10632
10633        let commands = worker
10634            .execute_workflow_task(workflow_task(
10635                "rust.typed-continue",
10636                Vec::new(),
10637                DEFAULT_CODEC,
10638            ))
10639            .expect("typed continue-as-new command");
10640
10641        assert_eq!(commands[0]["type"], "continue_as_new");
10642        assert_eq!(
10643            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10644                .expect("typed continue arguments"),
10645            AvroValue::Array(vec![typed_fidelity_probe()])
10646        );
10647    }
10648
10649    #[test]
10650    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10651        let client = Client::new("http://127.0.0.1:8080").expect("client");
10652        let mut worker = Worker::new(client, "rust-workers");
10653        worker.register_workflow("rust.continue", |ctx, _input| async move {
10654            ctx.continue_as_new(json!([2]))
10655        });
10656        let task = workflow_task(
10657            "rust.continue",
10658            vec![history_event(
10659                "WorkflowContinuedAsNew",
10660                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10661            )],
10662            JSON_CODEC,
10663        );
10664
10665        for _worker_restart_or_redelivery in 0..2 {
10666            let commands = worker
10667                .execute_workflow_task(task.clone())
10668                .expect("recorded transition replays");
10669            assert!(
10670                commands.is_empty(),
10671                "replay must not emit another successor"
10672            );
10673        }
10674    }
10675
10676    #[test]
10677    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10678        let ctx = workflow_context(Vec::new());
10679        let error = ctx
10680            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
10681            .expect_err("blank queue must be rejected");
10682
10683        let Error::InvalidContinueAsNewOptions(error) = error else {
10684            panic!("expected typed continue-as-new validation error");
10685        };
10686        assert_eq!(error.field, "task_queue");
10687        assert!(ctx.take_commands().expect("commands").is_empty());
10688    }
10689
10690    #[test]
10691    fn workflow_context_exposes_server_history_budget() {
10692        let client = Client::new("http://127.0.0.1:8080").expect("client");
10693        let mut worker = Worker::new(client, "rust-workers");
10694        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10695            let budget = ctx.history_budget()?;
10696            Ok(json!({
10697                "events": budget.event_count,
10698                "bytes": budget.size_bytes,
10699                "recommended": budget.continue_as_new_recommended,
10700                "pressure": budget.pressure,
10701            }))
10702        });
10703        let task: WorkflowTask = serde_json::from_value(json!({
10704            "task_id": "task-history-budget",
10705            "workflow_type": "rust.history-budget",
10706            "payload_codec": JSON_CODEC,
10707            "history_events": [],
10708            "total_history_events": 480,
10709            "history_size_bytes": 1_048_576,
10710            "continue_as_new_recommended": true,
10711            "history_budget_pressure": "continue_as_new_recommended",
10712        }))
10713        .expect("published workflow task");
10714
10715        let commands = worker
10716            .execute_workflow_task(task)
10717            .expect("history-budget workflow");
10718        let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10719        assert_eq!(result["events"], 480);
10720        assert_eq!(result["bytes"], 1_048_576);
10721        assert_eq!(result["recommended"], true);
10722        assert_eq!(result["pressure"], "continue_as_new_recommended");
10723    }
10724
10725    #[test]
10726    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10727        let client = Client::new("http://127.0.0.1:8080").expect("client");
10728        let mut worker = Worker::new(client, "rust-workers");
10729        worker.register_workflow("rust.failing", |_ctx, _input| async move {
10730            Err(Error::Codec("rust_conformance_failure".to_string()))
10731        });
10732        let task = WorkflowTask {
10733            task_id: "wft-rust-failing-1".to_string(),
10734            workflow_id: Some("wf-rust-failing".to_string()),
10735            run_id: Some("run-rust-failing".to_string()),
10736            workflow_type: "rust.failing".to_string(),
10737            payload_codec: JSON_CODEC.to_string(),
10738            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10739            history_events: Vec::new(),
10740            total_history_events: Some(0),
10741            history_size_bytes: None,
10742            continue_as_new_recommended: None,
10743            history_budget_pressure: None,
10744            next_history_page_token: None,
10745            workflow_task_attempt: 1,
10746            workflow_signal_id: None,
10747            signal_name: None,
10748            signal_arguments: None,
10749            workflow_update_id: None,
10750            update_name: None,
10751            lease_owner: Some("rust-worker".to_string()),
10752        };
10753
10754        let commands = worker
10755            .execute_workflow_task(task)
10756            .expect("handler failure becomes a workflow command");
10757
10758        assert_eq!(commands.len(), 1);
10759        assert_eq!(commands[0]["type"], "fail_workflow");
10760        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10761        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10762        assert_eq!(commands[0]["non_retryable"], false);
10763        assert_eq!(
10764            commands[0]["message"],
10765            "codec error: rust_conformance_failure"
10766        );
10767        assert_eq!(
10768            commands[0]["exception"]["message"],
10769            "codec error: rust_conformance_failure"
10770        );
10771    }
10772
10773    #[test]
10774    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10775        let client = Client::new("http://127.0.0.1:8080").expect("client");
10776        let mut worker = Worker::new(client, "rust-workers");
10777        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10778            let _: String = ctx.side_effect(|| "captured".to_string())?;
10779            Err(Error::WorkerLoop("application failure".to_string()))
10780        });
10781
10782        let commands = worker
10783            .execute_workflow_task(workflow_task(
10784                "rust.failing-after-side-effect",
10785                Vec::new(),
10786                JSON_CODEC,
10787            ))
10788            .expect("ordinary failure remains a workflow decision");
10789
10790        assert_eq!(commands.len(), 2);
10791        assert_eq!(commands[0]["type"], "record_side_effect");
10792        assert_eq!(commands[1]["type"], "fail_workflow");
10793    }
10794
10795    #[test]
10796    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10797        let client = Client::new("http://127.0.0.1:8080").expect("client");
10798        let mut worker = Worker::new(client, "rust-workers");
10799        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10800            Err(Error::WorkerLoop("application failure".to_string()))
10801        });
10802        let result =
10803            encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10804
10805        let error = worker
10806            .execute_workflow_task(workflow_task(
10807                "rust.removed-side-effect",
10808                vec![history_event(
10809                    "SideEffectRecorded",
10810                    json!({"sequence": 1, "result": result}),
10811                )],
10812                JSON_CODEC,
10813            ))
10814            .expect_err("removed committed history must not become fail_workflow");
10815
10816        let Error::NonDeterministicReplay(failure) = error else {
10817            panic!("expected typed replay failure");
10818        };
10819        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10820        assert_eq!(failure.sequence, Some(1));
10821        assert_eq!(failure.expected.as_deref(), Some("side effect"));
10822    }
10823
10824    #[test]
10825    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10826        let client = Client::new("http://127.0.0.1:8080").expect("client");
10827        let mut worker = Worker::new(client, "rust-workers");
10828        worker.register_workflow(
10829            "rust.side-effect-before-marker-error",
10830            |ctx, _input| async move {
10831                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10832                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10833                ctx.get_version("restart-safe", 2, 2)?;
10834                Ok(Value::Null)
10835            },
10836        );
10837
10838        let error = worker
10839            .execute_workflow_task(workflow_task(
10840                "rust.side-effect-before-marker-error",
10841                vec![history_event(
10842                    "VersionMarkerRecorded",
10843                    json!({
10844                        "sequence": 1,
10845                        "change_id": "restart-safe",
10846                        "version": 1,
10847                        "min_supported": 1,
10848                        "max_supported": 1,
10849                    }),
10850                )],
10851                JSON_CODEC,
10852            ))
10853            .expect_err("replay error must return no queued workflow commands");
10854
10855        let Error::NonDeterministicReplay(failure) = error else {
10856            panic!("expected typed replay failure");
10857        };
10858        assert_eq!(failure.reason, "version_marker_incompatible_range");
10859        assert_eq!(failure.sequence, Some(1));
10860    }
10861
10862    #[test]
10863    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10864        let client = Client::new("http://127.0.0.1:8080").expect("client");
10865        let mut worker = Worker::new(client, "rust-workers");
10866        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10867            ctx.sleep(Duration::from_secs(5)).await?;
10868            Ok(json!({"status": "timer fired"}))
10869        });
10870
10871        let task = WorkflowTask {
10872            task_id: "wft-rust-timer-pending".to_string(),
10873            workflow_id: Some("wf-rust-timer".to_string()),
10874            run_id: Some("run-rust-timer".to_string()),
10875            workflow_type: "rust.timer.pending".to_string(),
10876            payload_codec: JSON_CODEC.to_string(),
10877            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10878            history_events: vec![history_event(
10879                "TimerScheduled",
10880                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10881            )],
10882            total_history_events: Some(1),
10883            history_size_bytes: None,
10884            continue_as_new_recommended: None,
10885            history_budget_pressure: None,
10886            next_history_page_token: None,
10887            workflow_task_attempt: 1,
10888            workflow_signal_id: None,
10889            signal_name: None,
10890            signal_arguments: None,
10891            workflow_update_id: None,
10892            update_name: None,
10893            lease_owner: Some("rust-worker".to_string()),
10894        };
10895
10896        for _redelivery_or_restart in 0..2 {
10897            let commands = worker
10898                .execute_workflow_task(task.clone())
10899                .expect("recorded timer remains pending");
10900            assert!(
10901                commands.is_empty(),
10902                "recorded timer must not be rescheduled"
10903            );
10904        }
10905    }
10906
10907    #[test]
10908    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10909        let client = Client::new("http://127.0.0.1:8080").expect("client");
10910        let mut worker = Worker::new(client, "rust-workers");
10911        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10912            Ok(json!({"status": "completed"}))
10913        });
10914        let task = WorkflowTask {
10915            task_id: "wft-rust-timer-removed".to_string(),
10916            workflow_id: Some("wf-rust-timer".to_string()),
10917            run_id: Some("run-rust-timer".to_string()),
10918            workflow_type: "rust.timer.removed".to_string(),
10919            payload_codec: JSON_CODEC.to_string(),
10920            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10921            history_events: vec![
10922                history_event(
10923                    "TimerScheduled",
10924                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10925                ),
10926                history_event(
10927                    "TimerFired",
10928                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10929                ),
10930            ],
10931            total_history_events: Some(2),
10932            history_size_bytes: None,
10933            continue_as_new_recommended: None,
10934            history_budget_pressure: None,
10935            next_history_page_token: None,
10936            workflow_task_attempt: 1,
10937            workflow_signal_id: None,
10938            signal_name: None,
10939            signal_arguments: None,
10940            workflow_update_id: None,
10941            update_name: None,
10942            lease_owner: Some("rust-worker".to_string()),
10943        };
10944
10945        let Error::NonDeterministicReplay(failure) = worker
10946            .execute_workflow_task(task)
10947            .expect_err("removed timer must fail replay")
10948        else {
10949            panic!("expected typed replay failure");
10950        };
10951        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10952        assert_eq!(failure.sequence, Some(1));
10953    }
10954
10955    #[test]
10956    fn workflow_context_emits_explicit_child_workflow_contract() {
10957        let ctx = WorkflowContext {
10958            state: Arc::new(Mutex::new(
10959                WorkflowState::new_with_identity(
10960                    Vec::new(),
10961                    Some("wf-parent".to_string()),
10962                    Some("run-parent".to_string()),
10963                    "parent-workers".to_string(),
10964                    JSON_CODEC.to_string(),
10965                    None,
10966                )
10967                .expect("workflow state"),
10968            )),
10969        };
10970        let options = ChildWorkflowOptions::new("python-workers")
10971            .parent_close_policy(ParentClosePolicy::RequestCancel)
10972            .retry_policy(ChildWorkflowRetryPolicy {
10973                max_attempts: Some(3),
10974                backoff_seconds: vec![1, 5],
10975                non_retryable_error_types: vec!["ValidationError".to_string()],
10976            })
10977            .execution_timeout_seconds(600)
10978            .run_timeout_seconds(120);
10979        let mut call = Box::pin(ctx.start_child_workflow(
10980            "python.fulfil-order",
10981            options,
10982            json!([{"order_id": "order-42"}]),
10983        ));
10984        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10985
10986        assert!(matches!(
10987            call.as_mut().poll(&mut task_context),
10988            Poll::Pending
10989        ));
10990        let commands = ctx.take_commands().expect("commands");
10991        assert_eq!(commands.len(), 1);
10992        let command = &commands[0];
10993        assert_eq!(command["type"], "start_child_workflow");
10994        assert_eq!(command["workflow_type"], "python.fulfil-order");
10995        assert_eq!(command["queue"], "python-workers");
10996        assert_eq!(command["parent_close_policy"], "request_cancel");
10997        assert_eq!(command["retry_policy"]["max_attempts"], 3);
10998        assert_eq!(command["execution_timeout_seconds"], 600);
10999        assert_eq!(command["run_timeout_seconds"], 120);
11000        assert_eq!(
11001            decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
11002            json!([{"order_id": "order-42"}])
11003        );
11004    }
11005
11006    fn child_parent_worker() -> Worker {
11007        let client = Client::new("http://127.0.0.1:8080").expect("client");
11008        let mut worker = Worker::new(client, "rust-parent-workers");
11009        worker.register_workflow("rust.parent", |ctx, _input| async move {
11010            let child = ctx
11011                .start_child_workflow(
11012                    "python.child",
11013                    ChildWorkflowOptions::new("python-child-workers")
11014                        .parent_close_policy(ParentClosePolicy::Terminate),
11015                    json!([{"codec_probe": [1, true, "rust"]}]),
11016                )
11017                .await?;
11018            Ok(json!({
11019                "parent_workflow_id": child.parent.workflow_id,
11020                "parent_run_id": child.parent.run_id,
11021                "child_workflow_id": child.child.workflow_id,
11022                "child_run_id": child.child.run_id,
11023                "child_workflow_type": child.child_workflow_type,
11024                "result": child.result,
11025            }))
11026        });
11027        worker
11028    }
11029
11030    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11031        WorkflowTask {
11032            task_id: "wft-child-parent".to_string(),
11033            workflow_id: Some("wf-parent".to_string()),
11034            run_id: Some("run-parent".to_string()),
11035            workflow_type: "rust.parent".to_string(),
11036            payload_codec: JSON_CODEC.to_string(),
11037            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
11038            history_events: vec![
11039                HistoryEvent {
11040                    event_type: "ChildWorkflowScheduled".to_string(),
11041                    payload: json!({
11042                        "sequence": 1,
11043                        "child_call_id": "call-child",
11044                        "child_workflow_instance_id": "wf-child",
11045                        "child_workflow_run_id": "run-child",
11046                        "child_workflow_type": "python.child",
11047                    }),
11048                    raw: HashMap::new(),
11049                },
11050                HistoryEvent {
11051                    event_type: event_type.to_string(),
11052                    payload,
11053                    raw: HashMap::new(),
11054                },
11055            ],
11056            total_history_events: Some(2),
11057            history_size_bytes: None,
11058            continue_as_new_recommended: None,
11059            history_budget_pressure: None,
11060            next_history_page_token: None,
11061            workflow_task_attempt: 1,
11062            workflow_signal_id: None,
11063            signal_name: None,
11064            signal_arguments: None,
11065            workflow_update_id: None,
11066            update_name: None,
11067            lease_owner: Some("rust-worker".to_string()),
11068        }
11069    }
11070
11071    #[test]
11072    fn committed_child_result_replays_without_starting_a_duplicate() {
11073        let worker = child_parent_worker();
11074        let task = child_parent_task(
11075            "ChildRunCompleted",
11076            json!({
11077                "sequence": 1,
11078                "child_call_id": "call-child",
11079                "child_workflow_instance_id": "wf-child",
11080                "child_workflow_run_id": "run-child",
11081                "child_workflow_type": "python.child",
11082                "payload_codec": "json",
11083                "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
11084            }),
11085        );
11086
11087        for _restart in 0..2 {
11088            let commands = worker
11089                .execute_workflow_task(task.clone())
11090                .expect("replayed parent task");
11091            assert_eq!(commands.len(), 1);
11092            assert_eq!(commands[0]["type"], "complete_workflow");
11093            assert!(!commands
11094                .iter()
11095                .any(|command| command["type"] == "start_child_workflow"));
11096            let output =
11097                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11098            assert_eq!(output["parent_workflow_id"], "wf-parent");
11099            assert_eq!(output["parent_run_id"], "run-parent");
11100            assert_eq!(output["child_workflow_id"], "wf-child");
11101            assert_eq!(output["child_run_id"], "run-child");
11102            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11103        }
11104    }
11105
11106    #[test]
11107    fn typed_child_arguments_and_results_survive_replay() {
11108        let client = Client::new("http://127.0.0.1:8080").expect("client");
11109        let mut worker = Worker::new(client, "rust-parent-workers");
11110        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11111            let child = ctx
11112                .start_child_workflow_avro_value(
11113                    "python.typed-child",
11114                    ChildWorkflowOptions::new("python-workers"),
11115                    AvroValue::Array(vec![typed_fidelity_probe()]),
11116                )
11117                .await?;
11118            Ok(child.result)
11119        });
11120
11121        let initial = worker
11122            .execute_workflow_task(workflow_task(
11123                "rust.typed-parent",
11124                Vec::new(),
11125                DEFAULT_CODEC,
11126            ))
11127            .expect("typed child start");
11128        assert_eq!(initial[0]["type"], "start_child_workflow");
11129        assert_eq!(
11130            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11131                .expect("typed child arguments"),
11132            AvroValue::Array(vec![typed_fidelity_probe()])
11133        );
11134
11135        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11136            .expect("typed child result");
11137        let task = workflow_task(
11138            "rust.typed-parent",
11139            vec![
11140                history_event(
11141                    "ChildWorkflowScheduled",
11142                    json!({
11143                        "sequence": 1,
11144                        "child_call_id": "call-typed",
11145                        "child_workflow_instance_id": "wf-child",
11146                        "child_workflow_run_id": "run-child",
11147                        "child_workflow_type": "python.typed-child",
11148                    }),
11149                ),
11150                history_event(
11151                    "ChildRunCompleted",
11152                    json!({
11153                        "sequence": 1,
11154                        "child_call_id": "call-typed",
11155                        "child_workflow_instance_id": "wf-child",
11156                        "child_workflow_run_id": "run-child",
11157                        "child_workflow_type": "python.typed-child",
11158                        "payload_codec": DEFAULT_CODEC,
11159                        "result": result,
11160                    }),
11161                ),
11162            ],
11163            DEFAULT_CODEC,
11164        );
11165
11166        let commands = worker
11167            .execute_workflow_task(task)
11168            .expect("typed child replay");
11169        assert_eq!(commands[0]["type"], "complete_workflow");
11170        assert_eq!(
11171            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11172                .expect("typed parent result"),
11173            typed_fidelity_probe()
11174        );
11175    }
11176
11177    #[test]
11178    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11179        let worker = child_parent_worker();
11180        let mut task = child_parent_task("unused", Value::Null);
11181        task.history_events.truncate(1);
11182        task.total_history_events = Some(1);
11183
11184        for _redelivery_or_restart in 0..2 {
11185            let commands = worker
11186                .execute_workflow_task(task.clone())
11187                .expect("recorded child remains pending");
11188            assert!(
11189                commands.is_empty(),
11190                "recorded pending child must not be started again"
11191            );
11192        }
11193    }
11194
11195    #[test]
11196    fn child_cancellation_becomes_stable_parent_failure_command() {
11197        let worker = child_parent_worker();
11198        let task = child_parent_task(
11199            "ChildRunCancelled",
11200            json!({
11201                "sequence": 1,
11202                "child_workflow_instance_id": "wf-child",
11203                "child_workflow_run_id": "run-child",
11204                "child_workflow_type": "python.child",
11205                "failure_id": "failure-child",
11206                "failure_category": "cancelled",
11207                "message": "cancelled by parent-close policy",
11208            }),
11209        );
11210
11211        let commands = worker
11212            .execute_workflow_task(task)
11213            .expect("parent settlement");
11214        assert_eq!(commands.len(), 1);
11215        assert_eq!(commands[0]["type"], "fail_workflow");
11216        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11217        assert_eq!(
11218            commands[0]["exception"]["properties"]["reason"],
11219            "cancelled"
11220        );
11221        assert_eq!(
11222            commands[0]["exception"]["properties"]["child_workflow_run_id"],
11223            "run-child"
11224        );
11225    }
11226
11227    #[test]
11228    fn workflow_can_handle_typed_child_failure() {
11229        let client = Client::new("http://127.0.0.1:8080").expect("client");
11230        let mut worker = Worker::new(client, "rust-parent-workers");
11231        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11232            match ctx
11233                .start_child_workflow(
11234                    "python.child",
11235                    ChildWorkflowOptions::new("python-child-workers"),
11236                    json!([]),
11237                )
11238                .await
11239            {
11240                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11241                    "reason": failure.reason,
11242                    "failure_id": failure.failure_id,
11243                    "exception_class": failure.exception_class,
11244                    "child_run_id": failure.child_workflow_run_id,
11245                })),
11246                Err(error) => Err(error),
11247                Ok(_) => Err(Error::WorkerLoop(
11248                    "child unexpectedly succeeded".to_string(),
11249                )),
11250            }
11251        });
11252        let mut task = child_parent_task(
11253            "ChildRunFailed",
11254            json!({
11255                "sequence": 1,
11256                "child_workflow_instance_id": "wf-child",
11257                "child_workflow_run_id": "run-child",
11258                "child_workflow_type": "python.child",
11259                "failure_id": "failure-child",
11260                "failure_category": "child_workflow",
11261                "message": "payment rejected",
11262                "exception": {
11263                    "type": "PaymentRejected",
11264                    "class": "payments.PaymentRejected",
11265                    "message": "payment rejected"
11266                }
11267            }),
11268        );
11269        task.workflow_type = "rust.handled-parent".to_string();
11270
11271        let commands = worker.execute_workflow_task(task).expect("handled failure");
11272        assert_eq!(commands[0]["type"], "complete_workflow");
11273        let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11274        assert_eq!(output["reason"], "child_workflow");
11275        assert_eq!(output["failure_id"], "failure-child");
11276        assert_eq!(output["exception_class"], "payments.PaymentRejected");
11277        assert_eq!(output["child_run_id"], "run-child");
11278    }
11279
11280    #[test]
11281    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11282        let client = Client::new("http://127.0.0.1:8080").expect("client");
11283        let mut worker = Worker::new(client, "rust-workers");
11284
11285        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11286            let signal = ctx.wait_signal("start").await?;
11287            let name = signal
11288                .first()
11289                .and_then(|value| value.as_str())
11290                .unwrap_or("world");
11291            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11292            Ok(json!({
11293                "greeting": greeting,
11294                "language": "rust"
11295            }))
11296        });
11297
11298        let signal_arguments =
11299            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11300        let task = WorkflowTask {
11301            task_id: "wft-rust-signal-1".to_string(),
11302            workflow_id: Some("wf-rust-hello".to_string()),
11303            run_id: Some("run-rust-hello".to_string()),
11304            workflow_type: "rust.hello_workflow".to_string(),
11305            payload_codec: DEFAULT_CODEC.to_string(),
11306            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11307            history_events: vec![HistoryEvent {
11308                event_type: "SignalReceived".to_string(),
11309                payload: json!({
11310                    "signal_id": "sig-rust-1",
11311                    "signal_name": "start"
11312                }),
11313                raw: HashMap::new(),
11314            }],
11315            total_history_events: Some(1),
11316            history_size_bytes: None,
11317            continue_as_new_recommended: None,
11318            history_budget_pressure: None,
11319            next_history_page_token: None,
11320            workflow_task_attempt: 1,
11321            workflow_signal_id: Some("sig-rust-1".to_string()),
11322            signal_name: Some("start".to_string()),
11323            signal_arguments: Some(signal_arguments),
11324            workflow_update_id: None,
11325            update_name: None,
11326            lease_owner: Some("rust-worker".to_string()),
11327        };
11328
11329        let commands = worker.execute_workflow_task(task).expect("workflow task");
11330
11331        assert_eq!(commands.len(), 1);
11332        assert_eq!(commands[0]["type"], "schedule_activity");
11333        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11334        assert_eq!(
11335            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11336            json!(["Rust"])
11337        );
11338    }
11339
11340    #[test]
11341    fn workflow_task_appends_paginated_history_events() {
11342        let mut task = WorkflowTask {
11343            task_id: "wft-rust-pages-1".to_string(),
11344            workflow_id: Some("wf-rust-pages".to_string()),
11345            run_id: Some("run-rust-pages".to_string()),
11346            workflow_type: "rust.hello_workflow".to_string(),
11347            payload_codec: DEFAULT_CODEC.to_string(),
11348            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11349            history_events: vec![HistoryEvent {
11350                event_type: "WorkflowStarted".to_string(),
11351                payload: json!({}),
11352                raw: HashMap::new(),
11353            }],
11354            total_history_events: Some(3),
11355            history_size_bytes: None,
11356            continue_as_new_recommended: None,
11357            history_budget_pressure: None,
11358            next_history_page_token: Some("MQ==".to_string()),
11359            workflow_task_attempt: 1,
11360            workflow_signal_id: None,
11361            signal_name: None,
11362            signal_arguments: None,
11363            workflow_update_id: None,
11364            update_name: None,
11365            lease_owner: Some("rust-worker".to_string()),
11366        };
11367
11368        task.append_history_page(WorkflowTaskHistoryPage {
11369            history_events: vec![
11370                HistoryEvent {
11371                    event_type: "SignalReceived".to_string(),
11372                    payload: json!({
11373                        "signal_id": "sig-rust-1",
11374                        "signal_name": "start",
11375                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11376                            .expect("signal arguments")
11377                    }),
11378                    raw: HashMap::new(),
11379                },
11380                HistoryEvent {
11381                    event_type: "MarkerRecorded".to_string(),
11382                    payload: json!({"sequence": 3}),
11383                    raw: HashMap::new(),
11384                },
11385            ],
11386            total_history_events: Some(3),
11387            next_history_page_token: None,
11388        });
11389
11390        assert_eq!(task.history_events.len(), 3);
11391        assert_eq!(task.total_history_events, Some(3));
11392        assert_eq!(task.next_history_page_token, None);
11393
11394        let signal = task
11395            .history_events
11396            .iter()
11397            .find(|event| event.event_type == "SignalReceived")
11398            .expect("signal event");
11399        assert_eq!(
11400            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11401            vec![AvroValue::String("Rust".to_string())]
11402        );
11403    }
11404
11405    #[tokio::test]
11406    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11407        let client = Client::new("http://127.0.0.1:8080").expect("client");
11408        let mut worker = Worker::new(client, "rust-workers");
11409        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11410        worker.register_query("counter", "current", |ctx, _args| async move {
11411            let mut count = 0_i64;
11412            for signal in ctx.signal_events() {
11413                let value = signal
11414                    .arguments
11415                    .first()
11416                    .and_then(Value::as_i64)
11417                    .unwrap_or_default();
11418                match signal.name.as_str() {
11419                    "increment" => count += value,
11420                    "set" => count = value,
11421                    _ => {}
11422                }
11423            }
11424            Ok(json!(count))
11425        });
11426
11427        let task = QueryTask {
11428            query_task_id: "query-rust-counter".to_string(),
11429            query_task_attempt: 1,
11430            lease_owner: Some("rust-worker".to_string()),
11431            workflow_id: Some("counter-1".to_string()),
11432            run_id: Some("run-counter-1".to_string()),
11433            workflow_type: "counter".to_string(),
11434            query_name: "current".to_string(),
11435            payload_codec: DEFAULT_CODEC.to_string(),
11436            workflow_arguments: Some(
11437                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11438            ),
11439            query_arguments: Some(
11440                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11441            ),
11442            history_events: vec![
11443                HistoryEvent {
11444                    event_type: "SignalReceived".to_string(),
11445                    payload: json!({
11446                        "signal_id": "php-signal-1",
11447                        "signal_name": "increment",
11448                        "workflow_sequence": 1,
11449                        "payload_codec": DEFAULT_CODEC,
11450                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11451                    }),
11452                    raw: HashMap::new(),
11453                },
11454                HistoryEvent {
11455                    event_type: "SignalReceived".to_string(),
11456                    payload: json!({
11457                        "signal_id": "python-signal-2",
11458                        "signal_name": "increment",
11459                        "workflow_sequence": 2,
11460                        "payload_codec": JSON_CODEC,
11461                        "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11462                    }),
11463                    raw: HashMap::new(),
11464                },
11465                HistoryEvent {
11466                    event_type: "SignalReceived".to_string(),
11467                    payload: json!({
11468                        "signal_id": "rust-signal-3",
11469                        "signal_name": "set",
11470                        "workflow_sequence": 3,
11471                        "payload_codec": DEFAULT_CODEC,
11472                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11473                    }),
11474                    raw: HashMap::new(),
11475                },
11476            ],
11477            history_export: None,
11478            run_status: Some("completed".to_string()),
11479        };
11480
11481        let result = worker.execute_query_task(task).await.expect("query result");
11482        assert_eq!(result.into_json().expect("query projection"), json!(0));
11483    }
11484
11485    #[tokio::test]
11486    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11487        let worker = replay_counter_worker();
11488        let running_history = json!([
11489            {
11490                "type": "ActivityCompleted",
11491                "payload": {
11492                    "sequence": 1,
11493                    "activity_type": "load-counter",
11494                    "payload_codec": "json",
11495                    "result": {"codec": "json", "blob": "\"loaded\""}
11496                }
11497            },
11498            {
11499                "type": "SignalWaitOpened",
11500                "payload": {
11501                    "sequence": 3,
11502                    "signal_name": "increment"
11503                }
11504            },
11505            {
11506                "type": "SignalReceived",
11507                "payload": {
11508                    "signal_id": "signal-3",
11509                    "signal_name": "increment",
11510                    "workflow_sequence": 2,
11511                    "payload_codec": "json",
11512                    "arguments": {"codec": "json", "blob": "[3]"}
11513                }
11514            },
11515            {
11516                "type": "SignalApplied",
11517                "payload": {
11518                    "sequence": 3,
11519                    "signal_id": "signal-3",
11520                    "signal_name": "increment",
11521                    "payload_codec": "json",
11522                    "value": {"codec": "json", "blob": "[3]"}
11523                }
11524            }
11525        ]);
11526
11527        let running = worker
11528            .execute_query_task(replay_counter_query(
11529                "current",
11530                running_history.clone(),
11531                "running",
11532            ))
11533            .await
11534            .expect("running replay query");
11535        assert_eq!(
11536            running.clone().into_json().expect("query projection"),
11537            json!({"loaded": "loaded", "count": 3, "finished": false})
11538        );
11539
11540        let detached = worker
11541            .execute_query_task(replay_counter_query(
11542                "detached-mutation",
11543                running_history.clone(),
11544                "running",
11545            ))
11546            .await
11547            .expect("query mutates only its detached state clone");
11548        assert_eq!(detached.into_json().expect("query projection"), json!(999));
11549        let failed = worker
11550            .execute_query_task(replay_counter_query(
11551                "failed-mutation",
11552                running_history.clone(),
11553                "running",
11554            ))
11555            .await
11556            .expect_err("failed query");
11557        assert_eq!(failed.reason, "query_rejected");
11558        let unchanged = worker
11559            .execute_query_task(replay_counter_query("current", running_history, "running"))
11560            .await
11561            .expect("later query reconstructs unchanged state");
11562        assert_eq!(unchanged, running);
11563
11564        let restarted_worker = replay_counter_worker();
11565        let restarted_task: QueryTask = serde_json::from_value(json!({
11566            "query_task_id": "query-after-restart",
11567            "workflow_id": "counter-1",
11568            "run_id": "run-counter-1",
11569            "workflow_type": "replay-counter",
11570            "query_name": "current",
11571            "payload_codec": "json",
11572            "workflow_arguments": {"codec": "json", "blob": "[]"},
11573            "query_arguments": {"codec": "json", "blob": "[]"},
11574            "history_events": [],
11575            "history_export": {
11576                "payloads": {"codec": "json"},
11577                "history_events": [
11578                    {
11579                        "type": "ActivityCompleted",
11580                        "payload": {
11581                            "sequence": 1,
11582                            "activity_type": "load-counter",
11583                            "payload_codec": "json",
11584                            "result": null
11585                        }
11586                    },
11587                    {
11588                        "type": "SignalWaitOpened",
11589                        "payload": {
11590                            "sequence": 3,
11591                            "signal_name": "increment"
11592                        }
11593                    },
11594                    {
11595                        "type": "SignalReceived",
11596                        "payload": {
11597                            "signal_id": "signal-3",
11598                            "signal_name": "increment",
11599                            "workflow_sequence": 2
11600                        }
11601                    },
11602                    {
11603                        "type": "SignalApplied",
11604                        "payload": {
11605                            "sequence": 3,
11606                            "signal_id": "signal-3",
11607                            "signal_name": "increment"
11608                        }
11609                    },
11610                    {
11611                        "type": "SignalWaitOpened",
11612                        "payload": {
11613                            "sequence": 5,
11614                            "signal_name": "increment"
11615                        }
11616                    },
11617                    {
11618                        "type": "SignalReceived",
11619                        "payload": {
11620                            "signal_id": "signal-5",
11621                            "signal_name": "increment",
11622                            "workflow_sequence": 4
11623                        }
11624                    },
11625                    {
11626                        "type": "SignalApplied",
11627                        "payload": {
11628                            "sequence": 5,
11629                            "signal_id": "signal-5",
11630                            "signal_name": "increment"
11631                        }
11632                    }
11633                ],
11634                "activities": [{
11635                    "sequence": 1,
11636                    "activity_type": "load-counter",
11637                    "payload_codec": "json",
11638                    "result": {"codec": "json", "blob": "\"loaded\""}
11639                }],
11640                "signals": [
11641                    {
11642                        "id": "signal-3",
11643                        "name": "increment",
11644                        "workflow_sequence": 2,
11645                        "payload_codec": "json",
11646                        "arguments": "[3]"
11647                    },
11648                    {
11649                        "id": "signal-5",
11650                        "name": "increment",
11651                        "workflow_sequence": 4,
11652                        "payload_codec": "json",
11653                        "arguments": "[5]"
11654                    }
11655                ]
11656            },
11657            "run_status": "completed"
11658        }))
11659        .expect("cold replay query task");
11660        let completed = restarted_worker
11661            .execute_query_task(restarted_task)
11662            .await
11663            .expect("completed cold replay query");
11664        assert_eq!(
11665            completed.into_json().expect("query projection"),
11666            json!({"loaded": "loaded", "count": 8, "finished": true})
11667        );
11668    }
11669
11670    #[tokio::test]
11671    async fn replayed_query_replay_failures_are_machine_readable() {
11672        let worker = replay_counter_worker();
11673        let task = replay_counter_query(
11674            "current",
11675            json!([{
11676                "type": "ActivityCompleted",
11677                "payload": {
11678                    "sequence": 1,
11679                    "payload_codec": "json",
11680                    "result": {"codec": "json", "blob": "{"}
11681                }
11682            }]),
11683            "running",
11684        );
11685        let failure = worker
11686            .execute_query_task(task)
11687            .await
11688            .expect_err("invalid replay history payload");
11689        assert_eq!(failure.reason, "query_workflow_state_unavailable");
11690        assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11691    }
11692
11693    #[tokio::test]
11694    async fn query_task_restores_compact_history_from_export() {
11695        let client = Client::new("http://127.0.0.1:8080").expect("client");
11696        let mut worker = Worker::new(client, "rust-workers");
11697        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11698        worker.register_query("counter", "current", |ctx, _args| async move {
11699            Ok(json!(ctx.signals("increment")[0][0]))
11700        });
11701        let task: QueryTask = serde_json::from_value(json!({
11702            "query_task_id": "query-export",
11703            "workflow_type": "counter",
11704            "query_name": "current",
11705            "payload_codec": "json",
11706            "workflow_arguments": {"codec": "json", "blob": "[]"},
11707            "query_arguments": {"codec": "json", "blob": "[]"},
11708            "history_events": [],
11709            "history_export": {
11710                "payloads": {"codec": "json"},
11711                "history_events": [{
11712                    "type": "SignalReceived",
11713                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11714                }],
11715                "signals": [{
11716                    "id": "signal-export",
11717                    "name": "increment",
11718                    "status": "applied",
11719                    "workflow_sequence": 1,
11720                    "payload_codec": "json",
11721                    "arguments": "[9]"
11722                }]
11723            }
11724        }))
11725        .expect("query task");
11726
11727        let result = worker.execute_query_task(task).await.expect("query result");
11728        assert_eq!(result.into_json().expect("query projection"), json!(9));
11729    }
11730
11731    #[tokio::test]
11732    async fn query_task_failures_have_stable_reasons() {
11733        let client = Client::new("http://127.0.0.1:8080").expect("client");
11734        let mut worker = Worker::new(client, "rust-workers");
11735        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11736        worker.register_query(
11737            "counter",
11738            "current",
11739            |_ctx, _args| async move { Ok(json!(0)) },
11740        );
11741
11742        let base_task = QueryTask {
11743            query_task_id: "query-errors".to_string(),
11744            query_task_attempt: 1,
11745            lease_owner: None,
11746            workflow_id: Some("counter-errors".to_string()),
11747            run_id: Some("run-errors".to_string()),
11748            workflow_type: "counter".to_string(),
11749            query_name: "missing".to_string(),
11750            payload_codec: JSON_CODEC.to_string(),
11751            workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11752            query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11753            history_events: Vec::new(),
11754            history_export: None,
11755            run_status: Some("running".to_string()),
11756        };
11757
11758        let unknown = worker
11759            .execute_query_task(base_task.clone())
11760            .await
11761            .expect_err("unknown query");
11762        assert_eq!(unknown.reason, "rejected_unknown_query");
11763
11764        let mut malformed = base_task;
11765        malformed.query_name = "current".to_string();
11766        malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11767        let malformed = worker
11768            .execute_query_task(malformed)
11769            .await
11770            .expect_err("malformed payload");
11771        assert_eq!(malformed.reason, "query_payload_decode_failed");
11772
11773        let client = Client::new("http://127.0.0.1:8080").expect("client");
11774        let mut unavailable_worker = Worker::new(client, "rust-workers");
11775        unavailable_worker
11776            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11777        let unavailable_task: QueryTask = serde_json::from_value(json!({
11778            "query_task_id": "query-unavailable",
11779            "workflow_type": "counter",
11780            "query_name": "current",
11781            "payload_codec": "json",
11782            "workflow_arguments": {"codec": "json", "blob": "[]"},
11783            "query_arguments": {"codec": "json", "blob": "[]"}
11784        }))
11785        .expect("query task");
11786        let unavailable = unavailable_worker
11787            .execute_query_task(unavailable_task)
11788            .await
11789            .expect_err("query handler unavailable");
11790        assert_eq!(unavailable.reason, "query_handler_unavailable");
11791    }
11792
11793    #[tokio::test]
11794    async fn client_query_decodes_result_and_typed_failure() {
11795        let server = MockWorkerServer::start();
11796        let client = Client::builder(server.base_url())
11797            .timeout(Duration::from_secs(2))
11798            .build()
11799            .expect("client");
11800
11801        let result = client
11802            .query_workflow("counter-1", "current", json!([]))
11803            .await
11804            .expect("query result");
11805        assert_eq!(result, json!({"count": 8}));
11806
11807        let error = client
11808            .query_workflow("counter-1", "missing", json!([]))
11809            .await
11810            .expect_err("unknown query");
11811        let Error::QueryFailed(failure) = error else {
11812            panic!("expected typed query failure");
11813        };
11814        assert_eq!(failure.status, 404);
11815        assert_eq!(failure.reason, "rejected_unknown_query");
11816    }
11817
11818    #[tokio::test]
11819    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11820        let server = MockWorkerServer::start();
11821        let client = Client::builder(server.base_url())
11822            .timeout(Duration::from_secs(2))
11823            .build()
11824            .expect("client");
11825        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11826
11827        client
11828            .start_workflow(
11829                "typed.echo",
11830                "rust-workers",
11831                "typed-start",
11832                arguments.clone(),
11833            )
11834            .await
11835            .expect("typed workflow start");
11836        assert_eq!(
11837            decode_wire_avro_value(
11838                &server.request_body("/api/workflows")["input"],
11839                DEFAULT_CODEC,
11840            )
11841            .expect("typed start input"),
11842            arguments
11843        );
11844
11845        client
11846            .signal_workflow("typed-1", "changed", arguments.clone())
11847            .await
11848            .expect("typed signal");
11849        assert_eq!(
11850            decode_wire_avro_value(
11851                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11852                DEFAULT_CODEC,
11853            )
11854            .expect("typed signal input"),
11855            arguments
11856        );
11857
11858        assert_eq!(
11859            client
11860                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11861                .await
11862                .expect("typed query"),
11863            typed_fidelity_probe()
11864        );
11865        assert_eq!(
11866            decode_wire_avro_value(
11867                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11868                DEFAULT_CODEC,
11869            )
11870            .expect("typed query input"),
11871            arguments
11872        );
11873
11874        assert_eq!(
11875            client
11876                .update_workflow_avro_value(
11877                    "typed-1",
11878                    "replace",
11879                    arguments.clone(),
11880                    Some("typed-request"),
11881                )
11882                .await
11883                .expect("typed update"),
11884            typed_fidelity_probe()
11885        );
11886        let update = server.request_body("/api/workflows/typed-1/update/replace");
11887        assert_eq!(update["request_id"], "typed-request");
11888        assert_eq!(
11889            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11890            arguments
11891        );
11892
11893        let handle = WorkflowHandle {
11894            client: client.clone(),
11895            workflow_id: "typed-1".to_string(),
11896            run_id: Some("run-typed-1".to_string()),
11897            workflow_type: "typed.echo".to_string(),
11898        };
11899        assert_eq!(
11900            handle
11901                .result_avro_value(WorkflowResultOptions::default())
11902                .await
11903                .expect("typed workflow result"),
11904            typed_fidelity_probe()
11905        );
11906
11907        client
11908            .complete_activity_task(
11909                "activity-typed",
11910                "attempt-typed",
11911                "rust-worker",
11912                typed_fidelity_probe(),
11913                DEFAULT_CODEC,
11914            )
11915            .await
11916            .expect("typed activity completion");
11917        assert_eq!(
11918            decode_wire_avro_value(
11919                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11920                    ["result"],
11921                DEFAULT_CODEC,
11922            )
11923            .expect("typed activity result"),
11924            typed_fidelity_probe()
11925        );
11926        client
11927            .fail_activity_task(
11928                "activity-typed",
11929                "attempt-typed",
11930                "rust-worker",
11931                "typed failure",
11932                true,
11933            )
11934            .await
11935            .expect("activity failure");
11936    }
11937
11938    #[tokio::test]
11939    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11940        let server = MockWorkerServer::start();
11941        let client = Client::builder(server.base_url())
11942            .timeout(Duration::from_secs(2))
11943            .build()
11944            .expect("client");
11945
11946        let options = WorkflowCommandOptions::new()
11947            .reason("cleanup requested")
11948            .request_id("cancel-17");
11949        let cancelled = client
11950            .cancel_workflow("wf-lifecycle", options)
11951            .await
11952            .expect("instance cancellation");
11953        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11954        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11955        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11956        assert_eq!(
11957            server.request_body("/api/workflows/wf-lifecycle/cancel"),
11958            json!({"reason":"cleanup requested","request_id":"cancel-17"})
11959        );
11960
11961        let terminated = client
11962            .terminate_workflow(
11963                "wf-lifecycle",
11964                WorkflowCommandOptions::new().reason("forced stop"),
11965            )
11966            .await
11967            .expect("instance termination");
11968        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11969        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11970
11971        client
11972            .cancel_workflow_run(
11973                "wf-lifecycle",
11974                "run-current",
11975                WorkflowCommandOptions::default(),
11976            )
11977            .await
11978            .expect("selected run cancellation");
11979        client
11980            .terminate_workflow_run(
11981                "wf-lifecycle",
11982                "run-current",
11983                WorkflowCommandOptions::default(),
11984            )
11985            .await
11986            .expect("selected run termination");
11987
11988        for (command, error) in [
11989            (
11990                WorkflowCommandKind::Cancel,
11991                client
11992                    .cancel_workflow_run(
11993                        "wf-lifecycle",
11994                        "run-stale",
11995                        WorkflowCommandOptions::default(),
11996                    )
11997                    .await
11998                    .expect_err("stale cancellation must be rejected"),
11999            ),
12000            (
12001                WorkflowCommandKind::Terminate,
12002                client
12003                    .terminate_workflow_run(
12004                        "wf-lifecycle",
12005                        "run-stale",
12006                        WorkflowCommandOptions::default(),
12007                    )
12008                    .await
12009                    .expect_err("stale termination must be rejected"),
12010            ),
12011        ] {
12012            let Error::WorkflowCommandRejected(rejection) = error else {
12013                panic!("expected typed command rejection");
12014            };
12015            assert_eq!(rejection.command, command);
12016            assert_eq!(rejection.status, 409);
12017            assert_eq!(rejection.reason, "historical_run_command_rejected");
12018            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12019            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12020        }
12021    }
12022
12023    #[tokio::test]
12024    async fn workflow_start_options_send_server_enforced_deadlines() {
12025        let server = MockWorkerServer::start();
12026        let client = Client::builder(server.base_url())
12027            .timeout(Duration::from_secs(2))
12028            .build()
12029            .expect("client");
12030
12031        let handle = client
12032            .start_workflow_with_options(
12033                "rust.timeout",
12034                "rust-timeouts",
12035                "wf-start-options",
12036                WorkflowStartOptions::new()
12037                    .execution_timeout_seconds(30)
12038                    .run_timeout_seconds(1),
12039                json!([]),
12040            )
12041            .await
12042            .expect("workflow start");
12043
12044        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12045        let body = server.request_body("/api/workflows");
12046        assert_eq!(body["execution_timeout_seconds"], 30);
12047        assert_eq!(body["run_timeout_seconds"], 1);
12048
12049        let invalid = client
12050            .start_workflow_with_options(
12051                "rust.timeout",
12052                "rust-timeouts",
12053                "wf-invalid-options",
12054                WorkflowStartOptions::new()
12055                    .execution_timeout_seconds(1)
12056                    .run_timeout_seconds(2),
12057                json!([]),
12058            )
12059            .await
12060            .expect_err("invalid deadline ordering");
12061        assert!(invalid
12062            .to_string()
12063            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12064    }
12065
12066    #[tokio::test]
12067    async fn workflow_result_returns_each_typed_terminal_outcome() {
12068        let server = MockWorkerServer::start();
12069        let client = Client::builder(server.base_url())
12070            .timeout(Duration::from_secs(2))
12071            .build()
12072            .expect("client");
12073        let options = WorkflowResultOptions {
12074            poll_interval: Duration::ZERO,
12075            timeout: Duration::from_secs(1),
12076        };
12077
12078        let failed = WorkflowHandle {
12079            client: client.clone(),
12080            workflow_id: "wf-failed".to_string(),
12081            run_id: Some("run-failed".to_string()),
12082            workflow_type: "failure".to_string(),
12083        }
12084        .result(options)
12085        .await
12086        .expect_err("failed outcome");
12087        let Error::WorkflowFailed(failure) = failed else {
12088            panic!("expected WorkflowFailed");
12089        };
12090        assert_eq!(failure.workflow_id, "wf-failed");
12091        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12092        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12093        assert_eq!(failure.failure_category.as_deref(), Some("application"));
12094        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12095        assert_eq!(
12096            failure.exception_class.as_deref(),
12097            Some("billing::PaymentError")
12098        );
12099        assert_eq!(failure.non_retryable, Some(true));
12100
12101        for (workflow_id, expected_kind, expected_reason) in [
12102            (
12103                "wf-cancelled",
12104                WorkflowTerminalKind::Cancelled,
12105                "cleanup requested",
12106            ),
12107            (
12108                "wf-terminated",
12109                WorkflowTerminalKind::Terminated,
12110                "forced stop",
12111            ),
12112            (
12113                "wf-timed-out",
12114                WorkflowTerminalKind::TimedOut,
12115                "run_timeout",
12116            ),
12117        ] {
12118            let error = WorkflowHandle {
12119                client: client.clone(),
12120                workflow_id: workflow_id.to_string(),
12121                run_id: None,
12122                workflow_type: "terminal".to_string(),
12123            }
12124            .result(options)
12125            .await
12126            .expect_err("typed terminal outcome");
12127            let outcome = match error {
12128                Error::WorkflowCancelled(outcome) => outcome,
12129                Error::WorkflowTerminated(outcome) => outcome,
12130                Error::WorkflowTimedOut(outcome) => outcome,
12131                other => panic!("unexpected terminal error: {other}"),
12132            };
12133            assert_eq!(outcome.kind, expected_kind);
12134            assert_eq!(outcome.workflow_id, workflow_id);
12135            assert_eq!(outcome.reason, expected_reason);
12136        }
12137
12138        let wait_timeout = WorkflowHandle {
12139            client,
12140            workflow_id: "wf-waiting".to_string(),
12141            run_id: Some("run-waiting".to_string()),
12142            workflow_type: "waiting".to_string(),
12143        }
12144        .result(WorkflowResultOptions {
12145            poll_interval: Duration::ZERO,
12146            timeout: Duration::ZERO,
12147        })
12148        .await
12149        .expect_err("client wait timeout");
12150        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12151            panic!("expected typed client timeout");
12152        };
12153        assert_eq!(timeout.reason, "result_wait_timeout");
12154        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12155        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12156    }
12157
12158    #[tokio::test]
12159    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12160        let server = MockWorkerServer::start();
12161        let client = Client::builder(server.base_url())
12162            .timeout(Duration::from_secs(2))
12163            .build()
12164            .expect("client");
12165
12166        let handle = WorkflowHandle {
12167            client,
12168            workflow_id: "wf-selected".to_string(),
12169            run_id: Some("run-selected".to_string()),
12170            workflow_type: "selected".to_string(),
12171        };
12172        let options = WorkflowResultOptions {
12173            poll_interval: Duration::ZERO,
12174            timeout: Duration::from_secs(1),
12175        };
12176
12177        let current = handle
12178            .result(options)
12179            .await
12180            .expect("instance result follows the current run");
12181        assert_eq!(current, json!("current run output"));
12182
12183        let error = handle
12184            .result_selected_run(options)
12185            .await
12186            .expect_err("the selected run is cancelled even though the current run completed");
12187
12188        let Error::WorkflowCancelled(outcome) = error else {
12189            panic!("expected selected run cancellation");
12190        };
12191        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12192        assert_eq!(outcome.reason, "selected run cancelled");
12193        assert_eq!(
12194            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12195            1
12196        );
12197        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12198    }
12199
12200    #[tokio::test]
12201    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12202        let server = MockWorkerServer::draining_polls();
12203        let client = Client::builder(server.base_url())
12204            .timeout(Duration::from_secs(2))
12205            .build()
12206            .expect("client");
12207
12208        let workflow = client
12209            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12210            .await
12211            .expect("workflow drain response");
12212        let activity = client
12213            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12214            .await
12215            .expect("activity drain response");
12216        let query = client
12217            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12218            .await
12219            .expect("query drain response");
12220
12221        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12222            assert_eq!(
12223                outcome,
12224                WorkerPollOutcome::Stop {
12225                    poll_status: Some("draining".to_string()),
12226                    reason: Some("worker_draining".to_string()),
12227                }
12228            );
12229        }
12230
12231        assert!(client
12232            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12233            .await
12234            .expect("compatibility poll")
12235            .is_none());
12236    }
12237
12238    #[tokio::test]
12239    async fn managed_worker_honors_drain_stop_for_every_task_family() {
12240        let server = MockWorkerServer::draining_polls();
12241        let client = Client::builder(server.base_url())
12242            .timeout(Duration::from_secs(2))
12243            .build()
12244            .expect("client");
12245
12246        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12247            .worker_id("draining-workflow-worker")
12248            .poll_timeout(Duration::ZERO);
12249        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12250        workflow_worker
12251            .run()
12252            .await
12253            .expect("workflow drain is a clean stop");
12254
12255        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12256            .worker_id("draining-activity-worker")
12257            .poll_timeout(Duration::ZERO);
12258        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12259        activity_worker
12260            .run()
12261            .await
12262            .expect("activity drain is a clean stop");
12263
12264        let mut query_worker = Worker::new(client, "rust-workers")
12265            .worker_id("draining-query-worker")
12266            .poll_timeout(Duration::ZERO);
12267        query_worker.register_query("counter", "current", |_ctx, _args| async {
12268            Ok(Value::Null)
12269        });
12270        query_worker
12271            .run()
12272            .await
12273            .expect("query drain is a clean stop");
12274    }
12275
12276    #[tokio::test]
12277    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12278        let server = MockWorkerServer::start();
12279        let client = Client::builder(server.base_url())
12280            .timeout(Duration::from_secs(2))
12281            .build()
12282            .expect("client");
12283
12284        let heartbeat = client
12285            .heartbeat_activity_task(
12286                "activity-cancel",
12287                "attempt-cancel",
12288                "rust-worker",
12289                typed_fidelity_probe(),
12290            )
12291            .await
12292            .expect("cancellation heartbeat");
12293        assert!(heartbeat.cancel_requested);
12294        assert!(heartbeat.should_stop());
12295        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12296        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12297        let heartbeat_body =
12298            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12299        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12300        assert_eq!(
12301            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12302                .expect("typed heartbeat details"),
12303            typed_fidelity_probe()
12304        );
12305
12306        let error = client
12307            .complete_activity_task(
12308                "activity-cancel",
12309                "attempt-cancel",
12310                "rust-worker",
12311                json!({"late":true}),
12312                JSON_CODEC,
12313            )
12314            .await
12315            .expect_err("late completion must be refused");
12316        assert!(activity_task_rejection_is_final(&error));
12317        let Error::ActivityTaskRejected(rejection) = error else {
12318            panic!("expected typed activity rejection");
12319        };
12320        assert_eq!(rejection.status, 409);
12321        assert_eq!(rejection.reason, "run_cancelled");
12322        assert!(rejection.cancel_requested);
12323        assert_eq!(rejection.can_continue, Some(false));
12324    }
12325
12326    #[tokio::test]
12327    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12328        let server = MockWorkerServer::cancelled_activity();
12329        let client = Client::builder(server.base_url())
12330            .timeout(Duration::from_secs(2))
12331            .build()
12332            .expect("client");
12333        let cancellation_observed = Arc::new(AtomicBool::new(false));
12334        let observed = Arc::clone(&cancellation_observed);
12335        let mut worker = Worker::new(client.clone(), "rust-workers")
12336            .worker_id("rust-cancel-worker")
12337            .poll_timeout(Duration::from_millis(10));
12338        worker.register_activity("cancel-aware", move |ctx, _args| {
12339            let observed = Arc::clone(&observed);
12340            async move {
12341                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12342                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12343                Ok(json!({"late":"completion"}))
12344            }
12345        });
12346
12347        assert_eq!(
12348            worker.run_once().await.expect("cancelled attempt handled"),
12349            1
12350        );
12351        assert!(cancellation_observed.load(Ordering::SeqCst));
12352        assert_eq!(
12353            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12354            1
12355        );
12356
12357        let mut restarted = Worker::new(client, "rust-workers")
12358            .worker_id("rust-cancel-worker-restarted")
12359            .poll_timeout(Duration::from_millis(10));
12360        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12361        assert_eq!(
12362            restarted
12363                .run_once()
12364                .await
12365                .expect("replacement worker continues polling"),
12366            0
12367        );
12368    }
12369
12370    #[tokio::test]
12371    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12372        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12373        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12374        let client = Client::builder(server.base_url())
12375            .timeout(Duration::from_secs(2))
12376            .build()
12377            .expect("client");
12378
12379        let direct_error = client
12380            .complete_workflow_task(
12381                "workflow-timeout-task",
12382                "timeout-worker",
12383                3,
12384                vec![json!({"type": "complete_workflow", "result": null})],
12385            )
12386            .await
12387            .expect_err("the low-level client preserves the completion rejection");
12388        let Error::Http { status, body } = direct_error else {
12389            panic!("expected the original HTTP completion rejection");
12390        };
12391        assert_eq!(status, reqwest::StatusCode::CONFLICT);
12392        assert_eq!(
12393            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12394            "run_timed_out"
12395        );
12396
12397        let mut worker = Worker::new(client, "rust-workers")
12398            .worker_id("timeout-worker")
12399            .poll_timeout(Duration::from_millis(10));
12400        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12401            Ok(json!({"late": "result"}))
12402        });
12403
12404        assert_eq!(
12405            worker
12406                .run_once()
12407                .await
12408                .expect("authoritative selected-run timeout settles the tick"),
12409            1
12410        );
12411        assert_eq!(
12412            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12413            2,
12414            "both the direct client proof and managed worker must see the rejection"
12415        );
12416    }
12417
12418    #[tokio::test]
12419    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12420        for (name, status, response) in [
12421            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12422            (
12423                "command was recorded",
12424                "409 Conflict",
12425                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12426            ),
12427            (
12428                "lease conflict",
12429                "409 Conflict",
12430                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12431            ),
12432            (
12433                "nonterminal run",
12434                "409 Conflict",
12435                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12436            ),
12437            (
12438                "different selected run",
12439                "409 Conflict",
12440                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12441            ),
12442            (
12443                "different task attempt",
12444                "409 Conflict",
12445                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12446            ),
12447            (
12448                "authentication failure",
12449                "401 Unauthorized",
12450                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12451            ),
12452            (
12453                "authorization failure",
12454                "403 Forbidden",
12455                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12456            ),
12457            (
12458                "protocol failure",
12459                "400 Bad Request",
12460                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12461            ),
12462            (
12463                "malformed command",
12464                "422 Unprocessable Entity",
12465                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12466            ),
12467            (
12468                "transient server failure",
12469                "503 Service Unavailable",
12470                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12471            ),
12472        ] {
12473            let server = MockWorkerServer::workflow_completion(status, response);
12474            let client = Client::builder(server.base_url())
12475                .timeout(Duration::from_secs(2))
12476                .build()
12477                .expect("client");
12478            let mut worker = Worker::new(client, "rust-workers")
12479                .worker_id("timeout-worker")
12480                .poll_timeout(Duration::from_millis(10));
12481            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12482                Ok(json!({"late": "result"}))
12483            });
12484
12485            let error = worker
12486                .run_once()
12487                .await
12488                .expect_err(&format!("{name} must remain an error"));
12489            assert!(
12490                matches!(error, Error::Http { .. } | Error::Protocol(_)),
12491                "{name} returned an unexpected error variant: {error}"
12492            );
12493        }
12494    }
12495
12496    #[tokio::test]
12497    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
12498        let server = MockWorkerServer::start();
12499        let client = Client::builder(server.base_url())
12500            .worker_token(Some("worker-secret".to_string()))
12501            .namespace("orders")
12502            .timeout(Duration::from_secs(2))
12503            .build()
12504            .expect("client");
12505        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
12506
12507        let result = client
12508            .deregister_worker_registration("worker/α space")
12509            .await
12510            .expect("deregister worker registration");
12511
12512        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
12513        assert_eq!(
12514            server.worker_protocol_for(path).as_deref(),
12515            Some(WORKER_PROTOCOL_VERSION)
12516        );
12517        assert_eq!(server.control_protocol_for(path), None);
12518        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
12519        assert_eq!(
12520            server.authorization_for(path).as_deref(),
12521            Some("Bearer worker-secret")
12522        );
12523        assert_eq!(
12524            result,
12525            WorkerDeregistrationEnvelope {
12526                worker_id: "deregistered-worker".to_string(),
12527                outcome: "deregistered".to_string(),
12528                recovered_workflow_task_count: 2,
12529            }
12530        );
12531    }
12532
12533    #[tokio::test]
12534    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
12535        let server = MockWorkerServer::start();
12536        let control_only = Client::builder(server.base_url())
12537            .control_token(Some("control-secret".to_string()))
12538            .build()
12539            .expect("control client");
12540
12541        let error = control_only
12542            .register_worker("worker", "queue", vec![], vec![], 1, 1)
12543            .await
12544            .expect_err("control token must not authorize a worker request");
12545        assert!(matches!(
12546            error,
12547            Error::MissingRoleCredentials { role: "worker", .. }
12548        ));
12549        assert_eq!(server.request_count("/api/worker/register"), 0);
12550
12551        let worker_only = Client::builder(server.base_url())
12552            .worker_token(Some("worker-secret".to_string()))
12553            .build()
12554            .expect("worker client");
12555        let error = worker_only
12556            .health()
12557            .await
12558            .expect_err("worker token must not authorize a control request");
12559        assert!(matches!(
12560            error,
12561            Error::MissingRoleCredentials {
12562                role: "control",
12563                ..
12564            }
12565        ));
12566        assert_eq!(server.request_count("/api/health"), 0);
12567    }
12568
12569    #[tokio::test]
12570    async fn shared_token_supports_worker_and_control_planes() {
12571        let server = MockWorkerServer::start();
12572        let client = Client::builder(server.base_url())
12573            .token(Some("shared-secret".to_string()))
12574            .build()
12575            .expect("client");
12576
12577        client.health().await.expect("control request");
12578        client
12579            .register_worker("worker", "queue", vec![], vec![], 1, 1)
12580            .await
12581            .expect("worker request");
12582
12583        assert_eq!(
12584            server.authorization_for("/api/health").as_deref(),
12585            Some("Bearer shared-secret")
12586        );
12587        assert_eq!(
12588            server.control_protocol_for("/api/health").as_deref(),
12589            Some(CONTROL_PLANE_VERSION)
12590        );
12591        assert_eq!(
12592            server.authorization_for("/api/worker/register").as_deref(),
12593            Some("Bearer shared-secret")
12594        );
12595        assert_eq!(
12596            server
12597                .worker_protocol_for("/api/worker/register")
12598                .as_deref(),
12599            Some(WORKER_PROTOCOL_VERSION)
12600        );
12601    }
12602
12603    #[tokio::test]
12604    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12605        let server = MockWorkerServer::start();
12606        let client = Client::builder(server.base_url())
12607            .timeout(Duration::from_secs(2))
12608            .build()
12609            .expect("client");
12610
12611        client
12612            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12613            .await
12614            .expect("register");
12615        client
12616            .heartbeat_worker("capture-worker", 1, 1)
12617            .await
12618            .expect("heartbeat");
12619        client
12620            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12621            .await
12622            .expect("workflow poll");
12623        client
12624            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12625            .await
12626            .expect("activity poll");
12627
12628        for path in [
12629            "/api/worker/register",
12630            "/api/worker/heartbeat",
12631            "/api/worker/workflow-tasks/poll",
12632            "/api/worker/activity-tasks/poll",
12633        ] {
12634            assert_eq!(
12635                server.worker_protocol_for(path).as_deref(),
12636                Some(WORKER_PROTOCOL_VERSION),
12637                "unexpected protocol for {path}"
12638            );
12639        }
12640
12641        assert_eq!(
12642            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12643            1
12644        );
12645        assert_eq!(
12646            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12647            1
12648        );
12649        assert!(
12650            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12651                .as_str()
12652                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12653        );
12654        assert!(
12655            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12656                .as_str()
12657                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12658        );
12659    }
12660
12661    #[tokio::test]
12662    async fn query_task_endpoints_send_the_query_feature_protocol() {
12663        let server = MockWorkerServer::start();
12664        let client = Client::builder(server.base_url())
12665            .timeout(Duration::from_secs(2))
12666            .build()
12667            .expect("client");
12668
12669        client
12670            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12671            .await
12672            .expect("query poll");
12673        client
12674            .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12675            .await
12676            .expect("query complete");
12677        client
12678            .fail_query_task(
12679                "query-capture",
12680                "capture-worker",
12681                1,
12682                "failed",
12683                "query_rejected",
12684                "QueryFailed",
12685            )
12686            .await
12687            .expect("query fail");
12688
12689        for path in [
12690            "/api/worker/query-tasks/poll",
12691            "/api/worker/query-tasks/query-capture/complete",
12692            "/api/worker/query-tasks/query-capture/fail",
12693        ] {
12694            assert_eq!(
12695                server.worker_protocol_for(path).as_deref(),
12696                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12697                "unexpected protocol for {path}"
12698            );
12699        }
12700
12701        assert_eq!(
12702            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12703            1
12704        );
12705        assert!(
12706            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12707                .as_str()
12708                .is_some_and(|id| id.starts_with("rust-query-poll-"))
12709        );
12710    }
12711
12712    #[tokio::test]
12713    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12714        let server = MockWorkerServer::transient_worker_failures();
12715        let client = Client::builder(server.base_url())
12716            .timeout(Duration::from_secs(2))
12717            .build()
12718            .expect("client");
12719
12720        client
12721            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12722            .await
12723            .expect("workflow poll retry");
12724        client
12725            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12726            .await
12727            .expect("activity poll retry");
12728        client
12729            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12730            .await
12731            .expect("query poll retry");
12732
12733        for path in [
12734            "/api/worker/workflow-tasks/poll",
12735            "/api/worker/activity-tasks/poll",
12736            "/api/worker/query-tasks/poll",
12737        ] {
12738            let bodies = server.request_bodies(path);
12739            assert_eq!(bodies.len(), 2, "{path} must be retried once");
12740            assert_eq!(
12741                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12742                "{path} must preserve the request binding across retry"
12743            );
12744        }
12745    }
12746
12747    #[tokio::test]
12748    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12749        let server = MockWorkerServer::consecutive_poll_failures(2);
12750        let client = Client::builder(server.base_url())
12751            .timeout(Duration::from_secs(2))
12752            .build()
12753            .expect("client");
12754        let mut worker = Worker::new(client, "capture")
12755            .worker_id("capture-worker")
12756            .poll_timeout(Duration::from_millis(10))
12757            .retry_policy(WorkerRetryPolicy {
12758                max_retries: 2,
12759                initial_backoff: Duration::from_millis(1),
12760                max_backoff: Duration::from_millis(1),
12761            });
12762        worker.register_workflow(
12763            "capture.workflow",
12764            |_ctx, _input| async move { Ok(Value::Null) },
12765        );
12766        worker.register_activity(
12767            "capture.activity",
12768            |_ctx, _input| async move { Ok(Value::Null) },
12769        );
12770        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12771            Ok(Value::Null)
12772        });
12773
12774        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12775
12776        for path in [
12777            "/api/worker/workflow-tasks/poll",
12778            "/api/worker/activity-tasks/poll",
12779            "/api/worker/query-tasks/poll",
12780        ] {
12781            let bodies = server.request_bodies(path);
12782            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12783            assert!(
12784                bodies
12785                    .iter()
12786                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12787                "{path} must preserve one request binding across every retry"
12788            );
12789        }
12790    }
12791
12792    #[tokio::test]
12793    async fn query_protocol_rejection_from_older_server_is_typed() {
12794        let server = MockWorkerServer::reject_query_protocol();
12795        let client = Client::builder(server.base_url())
12796            .timeout(Duration::from_secs(2))
12797            .build()
12798            .expect("client");
12799
12800        let error = client
12801            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12802            .await
12803            .expect_err("server below query protocol floor must reject");
12804        let Error::Protocol(failure) = error else {
12805            panic!("expected typed protocol failure");
12806        };
12807
12808        assert_eq!(failure.status, 400);
12809        assert_eq!(failure.reason, "unsupported_protocol_version");
12810        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12811        assert_eq!(
12812            failure.requested_version.as_deref(),
12813            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12814        );
12815        assert_eq!(
12816            server
12817                .worker_protocol_for("/api/worker/query-tasks/poll")
12818                .as_deref(),
12819            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12820        );
12821    }
12822
12823    #[tokio::test]
12824    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12825        let server = MockWorkerServer::reject_query_protocol();
12826        let client = Client::builder(server.base_url())
12827            .timeout(Duration::from_secs(2))
12828            .build()
12829            .expect("client");
12830        let mut worker = Worker::new(client, "rust-workers")
12831            .worker_id("baseline-worker")
12832            .poll_timeout(Duration::from_millis(10));
12833
12834        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12835            Ok(Value::Null)
12836        });
12837
12838        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12839        assert_eq!(
12840            server
12841                .worker_protocol_for("/api/worker/workflow-tasks/poll")
12842                .as_deref(),
12843            Some(WORKER_PROTOCOL_VERSION)
12844        );
12845        assert_eq!(
12846            server.worker_protocol_for("/api/worker/query-tasks/poll"),
12847            None,
12848            "a worker without query handlers must not use the query-task endpoint"
12849        );
12850    }
12851
12852    #[tokio::test]
12853    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12854        let server = MockWorkerServer::reject_query_completion();
12855        let client = Client::builder(server.base_url())
12856            .timeout(Duration::from_secs(2))
12857            .build()
12858            .expect("client");
12859
12860        let error = client
12861            .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12862            .await
12863            .expect_err("expired completion must be rejected");
12864        let Error::QueryFailed(failure) = error else {
12865            panic!("expected typed query failure");
12866        };
12867        assert_eq!(failure.status, 409);
12868        assert_eq!(failure.reason, "query_task_timed_out");
12869
12870        let mut worker = Worker::new(client, "rust-workers")
12871            .worker_id("late-worker")
12872            .poll_timeout(Duration::from_millis(10));
12873        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12874        worker.register_query(
12875            "counter",
12876            "current",
12877            |_ctx, _args| async move { Ok(json!(8)) },
12878        );
12879
12880        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12881        assert_eq!(
12882            worker
12883                .run_once()
12884                .await
12885                .expect("worker continues after late completion"),
12886            0
12887        );
12888        assert_eq!(
12889            server.request_count("/api/worker/query-tasks/query-late/complete"),
12890            2
12891        );
12892        assert_eq!(
12893            server.request_count("/api/worker/query-tasks/query-late/fail"),
12894            0,
12895            "a server completion rejection must not be reported as an encoding failure"
12896        );
12897    }
12898
12899    #[tokio::test]
12900    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
12901        let server = MockWorkerServer::start();
12902        let client = Client::builder(server.base_url())
12903            .timeout(Duration::from_secs(2))
12904            .build()
12905            .expect("client");
12906        let mut worker = Worker::new(client, "rust-workers")
12907            .worker_id("joined-worker")
12908            .poll_timeout(Duration::from_millis(10));
12909        worker.register_workflow(
12910            "joined.workflow",
12911            |_ctx, _input| async move { Ok(Value::Null) },
12912        );
12913        worker.register_activity(
12914            "joined.activity",
12915            |_ctx, _input| async move { Ok(Value::Null) },
12916        );
12917        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
12918            Ok(Value::Null)
12919        });
12920
12921        worker
12922            .run_until(tokio::time::sleep(Duration::from_millis(20)))
12923            .await
12924            .expect("normal shutdown");
12925
12926        let deregistration_path = "/api/worker/registrations/mock-worker";
12927        assert_eq!(server.request_count(deregistration_path), 1);
12928        for poll_path in [
12929            "/api/worker/workflow-tasks/poll",
12930            "/api/worker/activity-tasks/poll",
12931            "/api/worker/query-tasks/poll",
12932        ] {
12933            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
12934        }
12935        assert_eq!(
12936            server.captured_paths().last().map(String::as_str),
12937            Some(deregistration_path),
12938            "deregistration must start only after every poller has joined"
12939        );
12940    }
12941
12942    #[tokio::test]
12943    async fn registration_failure_does_not_deregister() {
12944        let server = MockWorkerServer::rejected_registration();
12945        let client = Client::builder(server.base_url())
12946            .timeout(Duration::from_secs(2))
12947            .build()
12948            .expect("client");
12949        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
12950
12951        let error = worker
12952            .run_until(async {})
12953            .await
12954            .expect_err("registration must fail");
12955        assert!(matches!(
12956            error,
12957            Error::Http {
12958                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
12959                ..
12960            }
12961        ));
12962        assert!(server
12963            .captured_paths()
12964            .iter()
12965            .all(|path| !path.starts_with("/api/worker/registrations/")));
12966    }
12967
12968    #[tokio::test]
12969    async fn declined_registration_does_not_deregister() {
12970        let server = MockWorkerServer::declined_registration();
12971        let client = Client::builder(server.base_url())
12972            .timeout(Duration::from_secs(2))
12973            .build()
12974            .expect("client");
12975        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
12976
12977        let error = worker
12978            .run_until(async {})
12979            .await
12980            .expect_err("declined registration must fail");
12981        assert!(matches!(error, Error::WorkerLoop(_)));
12982        assert!(error.to_string().contains("was not accepted"));
12983        assert!(server
12984            .captured_paths()
12985            .iter()
12986            .all(|path| !path.starts_with("/api/worker/registrations/")));
12987    }
12988
12989    #[tokio::test]
12990    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
12991        let server = MockWorkerServer::rejected_deregistration();
12992        let client = Client::builder(server.base_url())
12993            .timeout(Duration::from_secs(2))
12994            .build()
12995            .expect("client");
12996        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
12997
12998        let error = worker
12999            .run_until(async {})
13000            .await
13001            .expect_err("deregistration must fail");
13002        assert!(matches!(
13003            error,
13004            Error::Http {
13005                status: reqwest::StatusCode::FORBIDDEN,
13006                ..
13007            }
13008        ));
13009        assert_eq!(
13010            server.request_count("/api/worker/registrations/mock-worker"),
13011            1
13012        );
13013    }
13014
13015    #[tokio::test]
13016    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13017        let server = MockWorkerServer::rejected_deregistration_protocol();
13018        let client = Client::builder(server.base_url())
13019            .timeout(Duration::from_secs(2))
13020            .build()
13021            .expect("client");
13022        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13023
13024        let error = worker
13025            .run_until(async {})
13026            .await
13027            .expect_err("protocol rejection must fail shutdown");
13028        let Error::Protocol(failure) = error else {
13029            panic!("expected typed protocol failure");
13030        };
13031        assert_eq!(failure.reason, "unsupported_protocol_version");
13032        assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13033        assert_eq!(
13034            server.request_count("/api/worker/registrations/mock-worker"),
13035            1
13036        );
13037    }
13038
13039    #[tokio::test]
13040    async fn primary_poller_error_retains_deregistration_failure_context() {
13041        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13042        let client = Client::builder(server.base_url())
13043            .timeout(Duration::from_secs(2))
13044            .build()
13045            .expect("client");
13046        let mut worker = Worker::new(client, "rust-workers")
13047            .worker_id("combined-failure")
13048            .poll_timeout(Duration::from_millis(10));
13049        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13050            Ok(Value::Null)
13051        });
13052
13053        let error = worker
13054            .run()
13055            .await
13056            .expect_err("worker and cleanup must fail");
13057        let summary = error.to_string();
13058        assert!(summary.contains("authentication_failed"));
13059        assert!(summary.contains("worker cannot deregister"));
13060        let Error::WorkerShutdown {
13061            primary,
13062            deregistration,
13063        } = error
13064        else {
13065            panic!("expected combined worker shutdown error");
13066        };
13067        assert!(matches!(
13068            *primary,
13069            Error::Http {
13070                status: reqwest::StatusCode::UNAUTHORIZED,
13071                ..
13072            }
13073        ));
13074        assert!(matches!(
13075            *deregistration,
13076            Error::Http {
13077                status: reqwest::StatusCode::FORBIDDEN,
13078                ..
13079            }
13080        ));
13081        assert_eq!(
13082            server.request_count("/api/worker/registrations/mock-worker"),
13083            1
13084        );
13085    }
13086
13087    #[tokio::test]
13088    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
13089        let server = MockWorkerServer::start();
13090        let client = Client::builder(server.base_url())
13091            .timeout(Duration::from_secs(2))
13092            .build()
13093            .expect("client");
13094        let mut worker = Worker::new(client, "rust-workers")
13095            .worker_id("activity-only-worker")
13096            .poll_timeout(Duration::from_millis(10));
13097
13098        worker.register_activity(
13099            "activity.only",
13100            |_ctx, _args| async move { Ok(Value::Null) },
13101        );
13102
13103        worker.run_until(async {}).await.expect("run worker");
13104    }
13105
13106    #[tokio::test]
13107    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
13108        let server = MockWorkerServer::start();
13109        let client = Client::builder(server.base_url())
13110            .timeout(Duration::from_secs(2))
13111            .build()
13112            .expect("client");
13113        let mut worker = Worker::new(client, "rust-workers")
13114            .worker_id("workflow-only-worker")
13115            .poll_timeout(Duration::from_millis(10));
13116
13117        worker.register_workflow(
13118            "workflow.only",
13119            |_ctx, _input| async move { Ok(Value::Null) },
13120        );
13121
13122        worker.run_until(async {}).await.expect("run worker");
13123    }
13124
13125    #[tokio::test]
13126    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
13127        let server = MockWorkerServer::start();
13128        let client = Client::builder(server.base_url())
13129            .timeout(Duration::from_secs(2))
13130            .build()
13131            .expect("client");
13132        let observations = Arc::new(Mutex::new(Vec::new()));
13133        let observed = Arc::clone(&observations);
13134        let mut worker = Worker::new(client, "rust-workers")
13135            .worker_id("observed-heartbeat-worker")
13136            .poll_timeout(Duration::from_millis(10))
13137            .on_worker_heartbeat(move |observation| {
13138                observed
13139                    .lock()
13140                    .expect("heartbeat observations")
13141                    .push(observation.clone());
13142            });
13143
13144        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
13145            Ok(Value::Null)
13146        });
13147        let acknowledged = Arc::clone(&observations);
13148        worker
13149            .run_until(async move {
13150                tokio::time::timeout(Duration::from_secs(2), async move {
13151                    loop {
13152                        if !acknowledged
13153                            .lock()
13154                            .expect("heartbeat observations")
13155                            .is_empty()
13156                        {
13157                            break;
13158                        }
13159                        tokio::time::sleep(Duration::from_millis(1)).await;
13160                    }
13161                })
13162                .await
13163                .expect("heartbeat acknowledgement within timeout");
13164            })
13165            .await
13166            .expect("run worker");
13167
13168        let observations = observations.lock().expect("heartbeat observations");
13169        let first = observations.first().expect("heartbeat acknowledgement");
13170        assert_eq!(first.worker_id, "observed-heartbeat-worker");
13171        assert_eq!(first.task_queue, "rust-workers");
13172        assert!(first.acknowledged_at_unix_millis > 0);
13173        assert_eq!(first.acknowledgement, json!({}));
13174    }
13175
13176    #[tokio::test]
13177    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
13178        let server = MockWorkerServer::delayed_heartbeat_worker();
13179        let client = Client::builder(server.base_url())
13180            .timeout(Duration::from_secs(3))
13181            .build()
13182            .expect("client");
13183        let observations = Arc::new(Mutex::new(Vec::new()));
13184        let observed = Arc::clone(&observations);
13185        let mut worker = Worker::new(client, "rust-snapshot-workers")
13186            .worker_id("rust-snapshot-worker")
13187            .poll_timeout(Duration::from_millis(10))
13188            .on_worker_heartbeat(move |observation| {
13189                observed
13190                    .lock()
13191                    .expect("heartbeat observations")
13192                    .push(observation.clone());
13193            });
13194
13195        worker.register_workflow("snapshot", |ctx, _input| async move {
13196            ctx.wait_signal("finish").await?;
13197            Ok(json!({"status": "finished"}))
13198        });
13199        worker.register_query("snapshot", "current", |ctx, _args| async move {
13200            Ok(json!(ctx
13201                .signals("increment")
13202                .iter()
13203                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13204                .sum::<i64>()))
13205        });
13206        worker.register_activity("cancel-aware", |_ctx, _args| async move {
13207            Ok(json!({"late": "completion"}))
13208        });
13209
13210        worker
13211            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
13212            .await
13213            .expect("delayed heartbeat must allow a clean worker shutdown");
13214
13215        let observations = observations.lock().expect("heartbeat observations");
13216        assert!(
13217            observations.len() >= 3,
13218            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
13219        );
13220        assert!(
13221            observations.windows(2).all(|pair| {
13222                pair[1].acknowledged_at_unix_millis
13223                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
13224                    >= 850
13225            }),
13226            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
13227        );
13228        drop(observations);
13229
13230        let heartbeat_times = server.request_times("/api/worker/heartbeat");
13231        let delayed_request_at = *heartbeat_times
13232            .get(1)
13233            .expect("intentionally delayed heartbeat request");
13234        let delay_window_start = delayed_request_at + Duration::from_millis(100);
13235        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
13236        for path in [
13237            "/api/worker/workflow-tasks/poll",
13238            "/api/worker/activity-tasks/poll",
13239            "/api/worker/query-tasks/poll",
13240        ] {
13241            assert!(
13242                server
13243                    .request_times(path)
13244                    .iter()
13245                    .any(|received_at| *received_at >= delay_window_start
13246                        && *received_at <= delay_window_end),
13247                "{path} must keep polling while a heartbeat acknowledgement is delayed"
13248            );
13249        }
13250        assert!(
13251            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
13252            "workflow work must be settled"
13253        );
13254        assert!(
13255            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
13256            "activity work must be settled"
13257        );
13258        assert!(
13259            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
13260            "query work must be settled"
13261        );
13262    }
13263
13264    #[tokio::test]
13265    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
13266        let server = MockWorkerServer::heartbeat_retry_worker();
13267        let client = Client::builder(server.base_url())
13268            .timeout(Duration::from_secs(2))
13269            .build()
13270            .expect("client");
13271        let observations = Arc::new(Mutex::new(Vec::new()));
13272        let observed = Arc::clone(&observations);
13273        let worker = Worker::new(client, "rust-workers")
13274            .worker_id("heartbeat-retry-worker")
13275            .retry_policy(WorkerRetryPolicy {
13276                max_retries: 1,
13277                initial_backoff: Duration::from_millis(300),
13278                max_backoff: Duration::from_millis(300),
13279            })
13280            .on_worker_heartbeat(move |observation| {
13281                observed
13282                    .lock()
13283                    .expect("heartbeat observations")
13284                    .push(observation.clone());
13285            });
13286
13287        worker
13288            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
13289            .await
13290            .expect("retryable heartbeat failure must remain bounded and recover");
13291
13292        let observations = observations.lock().expect("heartbeat observations");
13293        assert!(observations.len() >= 3, "heartbeat retry must recover");
13294        assert!(
13295            observations.windows(2).all(|pair| {
13296                pair[1]
13297                    .acknowledged_at_unix_millis
13298                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
13299                    >= 850
13300            }),
13301            "a successful retry must start a fresh advertised cadence: {observations:?}"
13302        );
13303        assert_eq!(
13304            server.request_count("/api/worker/heartbeat"),
13305            observations.len() + 1,
13306            "one retryable failure must add exactly one bounded request"
13307        );
13308    }
13309
13310    #[tokio::test]
13311    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
13312        let server = MockWorkerServer::waiting_query_worker();
13313        let client = Client::builder(server.base_url())
13314            .timeout(Duration::from_secs(2))
13315            .build()
13316            .expect("client");
13317        let observations = Arc::new(Mutex::new(Vec::new()));
13318        let observed = Arc::clone(&observations);
13319        let mut worker = Worker::new(client, "rust-snapshot-workers")
13320            .worker_id("rust-snapshot-worker")
13321            .poll_timeout(Duration::from_millis(10))
13322            .on_worker_heartbeat(move |observation| {
13323                observed
13324                    .lock()
13325                    .expect("heartbeat observations")
13326                    .push(observation.clone());
13327            });
13328
13329        worker.register_workflow("snapshot", |ctx, _input| async move {
13330            ctx.wait_signal("finish").await?;
13331            Ok(json!({"status": "finished"}))
13332        });
13333        worker.register_query("snapshot", "current", |ctx, _args| async move {
13334            let current = ctx
13335                .signals("increment")
13336                .iter()
13337                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13338                .sum::<i64>();
13339            Ok(json!(current))
13340        });
13341        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
13342
13343        worker
13344            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
13345            .await
13346            .expect("pending workflow and query poller must remain live until shutdown");
13347
13348        assert!(
13349            observations.lock().expect("heartbeat observations").len() >= 4,
13350            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
13351        );
13352        assert!(
13353            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
13354            "workflow polling must continue after empty replay acknowledgements"
13355        );
13356        assert!(
13357            server.request_count("/api/worker/query-tasks/poll") >= 2,
13358            "query polling must continue after serving the current query"
13359        );
13360        assert_eq!(
13361            server.request_body("/api/worker/register")["capabilities"],
13362            json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
13363        );
13364        assert_eq!(
13365            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
13366            json!({
13367                "queries": ["current"],
13368                "updates": ["replace"],
13369            })
13370        );
13371
13372        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
13373        assert_eq!(
13374            opened["commands"],
13375            json!([{
13376                "type": "open_signal_wait",
13377                "signal_name": "finish",
13378            }])
13379        );
13380
13381        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
13382            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
13383            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
13384            let failure = server.request_body(&fail_path);
13385            assert_eq!(
13386                failure["failure"]["type"],
13387                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
13388            );
13389            assert_eq!(server.request_count(&completion_path), 0);
13390        }
13391
13392        let query_completion =
13393            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
13394        assert_eq!(query_completion["result"], json!(8));
13395
13396        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
13397        assert_eq!(
13398            server.request_count(terminal_path),
13399            1,
13400            "the matching signal must settle the workflow exactly once"
13401        );
13402        let terminal = server.request_body(terminal_path);
13403        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
13404        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
13405        assert_eq!(
13406            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
13407                .expect("terminal workflow result"),
13408            json!({"status": "finished"})
13409        );
13410    }
13411
13412    #[tokio::test]
13413    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
13414        let server = MockWorkerServer::transient_worker_failures();
13415        let client = Client::builder(server.base_url())
13416            .timeout(Duration::from_secs(2))
13417            .build()
13418            .expect("client");
13419        let mut worker = Worker::new(client, "rust-workers")
13420            .worker_id("retry-worker")
13421            .poll_timeout(Duration::from_millis(10))
13422            .retry_policy(WorkerRetryPolicy {
13423                max_retries: 2,
13424                initial_backoff: Duration::from_millis(1),
13425                max_backoff: Duration::from_millis(1),
13426            });
13427        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13428        worker.register_activity(
13429            "counter.activity",
13430            |_ctx, _input| async move { Ok(Value::Null) },
13431        );
13432        worker.register_query(
13433            "counter",
13434            "current",
13435            |_ctx, _args| async move { Ok(json!(8)) },
13436        );
13437
13438        worker
13439            .run_until(tokio::time::sleep(Duration::from_millis(75)))
13440            .await
13441            .expect("transient failures must not stop the worker");
13442
13443        for path in [
13444            "/api/worker/heartbeat",
13445            "/api/worker/workflow-tasks/poll",
13446            "/api/worker/activity-tasks/poll",
13447            "/api/worker/query-tasks/poll",
13448        ] {
13449            assert!(
13450                server.request_count(path) >= 2,
13451                "{path} must continue after its transient failure"
13452            );
13453        }
13454    }
13455
13456    #[tokio::test]
13457    async fn worker_bounds_transport_retries() {
13458        let server = MockWorkerServer::unavailable_polls();
13459        let client = Client::builder(server.base_url())
13460            .timeout(Duration::from_secs(2))
13461            .build()
13462            .expect("client");
13463        let mut worker = Worker::new(client, "rust-workers")
13464            .worker_id("bounded-retry-worker")
13465            .poll_timeout(Duration::from_millis(10))
13466            .retry_policy(WorkerRetryPolicy {
13467                max_retries: 2,
13468                initial_backoff: Duration::from_millis(1),
13469                max_backoff: Duration::from_millis(1),
13470            });
13471        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13472
13473        let error = worker.run().await.expect_err("retry bound must terminate");
13474        assert!(matches!(error, Error::Transport(_)));
13475        assert_eq!(
13476            server.request_count("/api/worker/workflow-tasks/poll"),
13477            3,
13478            "one initial request plus exactly two retries"
13479        );
13480    }
13481
13482    #[tokio::test]
13483    async fn worker_retry_policy_can_disable_poll_retries() {
13484        let server = MockWorkerServer::unavailable_polls();
13485        let client = Client::builder(server.base_url())
13486            .timeout(Duration::from_secs(2))
13487            .build()
13488            .expect("client");
13489        let mut worker = Worker::new(client, "rust-workers")
13490            .worker_id("no-retry-worker")
13491            .poll_timeout(Duration::from_millis(10))
13492            .retry_policy(WorkerRetryPolicy {
13493                max_retries: 0,
13494                initial_backoff: Duration::from_millis(1),
13495                max_backoff: Duration::from_millis(1),
13496            });
13497        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13498
13499        let error = worker
13500            .run_once()
13501            .await
13502            .expect_err("disabled retries must return the first transport failure");
13503        assert!(matches!(error, Error::Transport(_)));
13504        assert_eq!(
13505            server.request_count("/api/worker/workflow-tasks/poll"),
13506            1,
13507            "max_retries=0 must send only the initial request"
13508        );
13509    }
13510
13511    #[tokio::test]
13512    async fn worker_does_not_retry_authentication_failures() {
13513        let server = MockWorkerServer::unauthorized_polls();
13514        let client = Client::builder(server.base_url())
13515            .timeout(Duration::from_secs(2))
13516            .build()
13517            .expect("client");
13518        let mut worker = Worker::new(client, "rust-workers")
13519            .worker_id("unauthorized-worker")
13520            .poll_timeout(Duration::from_millis(10));
13521        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13522
13523        let error = worker
13524            .run()
13525            .await
13526            .expect_err("authentication must terminate");
13527        let Error::Http { status, body } = error else {
13528            panic!("expected stable HTTP authentication error");
13529        };
13530        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13531        assert!(body.contains("authentication_failed"));
13532        assert_eq!(
13533            server.request_count("/api/worker/workflow-tasks/poll"),
13534            1,
13535            "authentication failures must not be retried"
13536        );
13537    }
13538
13539    #[derive(Clone, Debug)]
13540    struct CapturedRequest {
13541        method: String,
13542        path: String,
13543        authorization: Option<String>,
13544        namespace: Option<String>,
13545        worker_protocol: Option<String>,
13546        control_protocol: Option<String>,
13547        body: String,
13548        received_at: Instant,
13549    }
13550
13551    struct MockWorkerServer {
13552        addr: SocketAddr,
13553        stop: Arc<AtomicBool>,
13554        requests: Arc<Mutex<Vec<CapturedRequest>>>,
13555        thread: Option<thread::JoinHandle<()>>,
13556    }
13557
13558    #[derive(Clone, Copy, Default)]
13559    struct MockWorkerBehavior {
13560        reject_query_protocol: bool,
13561        reject_query_completion: bool,
13562        waiting_query_worker: bool,
13563        decline_registration: bool,
13564        complete_named_signal: bool,
13565        poll_failures_per_path: usize,
13566        heartbeat_failures: usize,
13567        heartbeat_failure_request: Option<usize>,
13568        delayed_heartbeat_request: Option<usize>,
13569        heartbeat_response_delay: Duration,
13570        concurrent_requests: bool,
13571        unauthorized_polls: bool,
13572        reject_registration: bool,
13573        reject_deregistration: bool,
13574        reject_deregistration_protocol: bool,
13575        cancelled_activity: bool,
13576        draining_polls: bool,
13577        workflow_completion_status: Option<&'static str>,
13578        workflow_completion_body: Option<&'static str>,
13579    }
13580
13581    impl MockWorkerServer {
13582        fn start() -> Self {
13583            Self::start_with_behavior(MockWorkerBehavior::default())
13584        }
13585
13586        fn reject_query_protocol() -> Self {
13587            Self::start_with_behavior(MockWorkerBehavior {
13588                reject_query_protocol: true,
13589                ..MockWorkerBehavior::default()
13590            })
13591        }
13592
13593        fn reject_query_completion() -> Self {
13594            Self::start_with_behavior(MockWorkerBehavior {
13595                reject_query_completion: true,
13596                ..MockWorkerBehavior::default()
13597            })
13598        }
13599
13600        fn waiting_query_worker() -> Self {
13601            Self::start_with_behavior(MockWorkerBehavior {
13602                waiting_query_worker: true,
13603                complete_named_signal: true,
13604                ..MockWorkerBehavior::default()
13605            })
13606        }
13607
13608        fn transient_worker_failures() -> Self {
13609            Self::start_with_behavior(MockWorkerBehavior {
13610                poll_failures_per_path: 1,
13611                heartbeat_failures: 1,
13612                ..MockWorkerBehavior::default()
13613            })
13614        }
13615
13616        fn consecutive_poll_failures(count: usize) -> Self {
13617            Self::start_with_behavior(MockWorkerBehavior {
13618                poll_failures_per_path: count,
13619                ..MockWorkerBehavior::default()
13620            })
13621        }
13622
13623        fn delayed_heartbeat_worker() -> Self {
13624            Self::start_with_behavior(MockWorkerBehavior {
13625                waiting_query_worker: true,
13626                delayed_heartbeat_request: Some(2),
13627                heartbeat_response_delay: Duration::from_millis(1_500),
13628                concurrent_requests: true,
13629                cancelled_activity: true,
13630                ..MockWorkerBehavior::default()
13631            })
13632        }
13633
13634        fn heartbeat_retry_worker() -> Self {
13635            Self::start_with_behavior(MockWorkerBehavior {
13636                waiting_query_worker: true,
13637                heartbeat_failure_request: Some(2),
13638                concurrent_requests: true,
13639                ..MockWorkerBehavior::default()
13640            })
13641        }
13642
13643        fn unavailable_polls() -> Self {
13644            Self::start_with_behavior(MockWorkerBehavior {
13645                poll_failures_per_path: usize::MAX,
13646                ..MockWorkerBehavior::default()
13647            })
13648        }
13649
13650        fn unauthorized_polls() -> Self {
13651            Self::start_with_behavior(MockWorkerBehavior {
13652                unauthorized_polls: true,
13653                ..MockWorkerBehavior::default()
13654            })
13655        }
13656
13657        fn rejected_registration() -> Self {
13658            Self::start_with_behavior(MockWorkerBehavior {
13659                reject_registration: true,
13660                ..MockWorkerBehavior::default()
13661            })
13662        }
13663
13664        fn declined_registration() -> Self {
13665            Self::start_with_behavior(MockWorkerBehavior {
13666                decline_registration: true,
13667                ..MockWorkerBehavior::default()
13668            })
13669        }
13670
13671        fn rejected_deregistration() -> Self {
13672            Self::start_with_behavior(MockWorkerBehavior {
13673                reject_deregistration: true,
13674                ..MockWorkerBehavior::default()
13675            })
13676        }
13677
13678        fn rejected_deregistration_protocol() -> Self {
13679            Self::start_with_behavior(MockWorkerBehavior {
13680                reject_deregistration_protocol: true,
13681                ..MockWorkerBehavior::default()
13682            })
13683        }
13684
13685        fn unauthorized_polls_and_rejected_deregistration() -> Self {
13686            Self::start_with_behavior(MockWorkerBehavior {
13687                unauthorized_polls: true,
13688                reject_deregistration: true,
13689                ..MockWorkerBehavior::default()
13690            })
13691        }
13692
13693        fn cancelled_activity() -> Self {
13694            Self::start_with_behavior(MockWorkerBehavior {
13695                cancelled_activity: true,
13696                ..MockWorkerBehavior::default()
13697            })
13698        }
13699
13700        fn draining_polls() -> Self {
13701            Self::start_with_behavior(MockWorkerBehavior {
13702                draining_polls: true,
13703                ..MockWorkerBehavior::default()
13704            })
13705        }
13706
13707        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13708            Self::start_with_behavior(MockWorkerBehavior {
13709                workflow_completion_status: Some(status),
13710                workflow_completion_body: Some(body),
13711                ..MockWorkerBehavior::default()
13712            })
13713        }
13714
13715        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13716            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13717            listener
13718                .set_nonblocking(true)
13719                .expect("configure mock listener");
13720            let addr = listener.local_addr().expect("mock server address");
13721            let stop = Arc::new(AtomicBool::new(false));
13722            let server_stop = Arc::clone(&stop);
13723            let requests = Arc::new(Mutex::new(Vec::new()));
13724            let server_requests = Arc::clone(&requests);
13725            let thread = thread::spawn(move || {
13726                let mut request_threads = Vec::new();
13727                while !server_stop.load(Ordering::SeqCst) {
13728                    match listener.accept() {
13729                        Ok((mut stream, _)) => {
13730                            if behavior.concurrent_requests {
13731                                let requests = Arc::clone(&server_requests);
13732                                request_threads.push(thread::spawn(move || {
13733                                    handle_mock_worker_request(&mut stream, &requests, behavior)
13734                                }));
13735                            } else {
13736                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
13737                            }
13738                        }
13739                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13740                            let mut index = 0;
13741                            while index < request_threads.len() {
13742                                if request_threads[index].is_finished() {
13743                                    request_threads
13744                                        .swap_remove(index)
13745                                        .join()
13746                                        .expect("join mock request");
13747                                } else {
13748                                    index += 1;
13749                                }
13750                            }
13751                            thread::sleep(Duration::from_millis(5));
13752                        }
13753                        Err(_) => break,
13754                    }
13755                }
13756                for request_thread in request_threads {
13757                    request_thread.join().expect("join mock request");
13758                }
13759            });
13760
13761            Self {
13762                addr,
13763                stop,
13764                requests,
13765                thread: Some(thread),
13766            }
13767        }
13768
13769        fn base_url(&self) -> String {
13770            format!("http://{}", self.addr)
13771        }
13772
13773        fn worker_protocol_for(&self, path: &str) -> Option<String> {
13774            self.requests
13775                .lock()
13776                .expect("captured requests")
13777                .iter()
13778                .find(|request| request.path == path)
13779                .and_then(|request| request.worker_protocol.clone())
13780        }
13781
13782        fn control_protocol_for(&self, path: &str) -> Option<String> {
13783            self.requests
13784                .lock()
13785                .expect("captured requests")
13786                .iter()
13787                .find(|request| request.path == path)
13788                .and_then(|request| request.control_protocol.clone())
13789        }
13790
13791        fn method_for(&self, path: &str) -> Option<String> {
13792            self.requests
13793                .lock()
13794                .expect("captured requests")
13795                .iter()
13796                .find(|request| request.path == path)
13797                .map(|request| request.method.clone())
13798        }
13799
13800        fn authorization_for(&self, path: &str) -> Option<String> {
13801            self.requests
13802                .lock()
13803                .expect("captured requests")
13804                .iter()
13805                .find(|request| request.path == path)
13806                .and_then(|request| request.authorization.clone())
13807        }
13808
13809        fn namespace_for(&self, path: &str) -> Option<String> {
13810            self.requests
13811                .lock()
13812                .expect("captured requests")
13813                .iter()
13814                .find(|request| request.path == path)
13815                .and_then(|request| request.namespace.clone())
13816        }
13817
13818        fn request_count(&self, path: &str) -> usize {
13819            self.requests
13820                .lock()
13821                .expect("captured requests")
13822                .iter()
13823                .filter(|request| request.path == path)
13824                .count()
13825        }
13826
13827        fn captured_paths(&self) -> Vec<String> {
13828            self.requests
13829                .lock()
13830                .expect("captured requests")
13831                .iter()
13832                .map(|request| request.path.clone())
13833                .collect()
13834        }
13835
13836        fn request_times(&self, path: &str) -> Vec<Instant> {
13837            self.requests
13838                .lock()
13839                .expect("captured requests")
13840                .iter()
13841                .filter(|request| request.path == path)
13842                .map(|request| request.received_at)
13843                .collect()
13844        }
13845
13846        fn request_body(&self, path: &str) -> Value {
13847            let requests = self.requests.lock().expect("captured requests");
13848            let body = &requests
13849                .iter()
13850                .find(|request| request.path == path)
13851                .unwrap_or_else(|| panic!("missing request for {path}"))
13852                .body;
13853            serde_json::from_str(body).unwrap_or_else(|error| {
13854                panic!("invalid JSON request body for {path}: {error}: {body:?}")
13855            })
13856        }
13857
13858        fn request_bodies(&self, path: &str) -> Vec<Value> {
13859            self.requests
13860                .lock()
13861                .expect("captured requests")
13862                .iter()
13863                .filter(|request| request.path == path)
13864                .map(|request| {
13865                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
13866                        panic!(
13867                            "invalid JSON request body for {path}: {error}: {:?}",
13868                            request.body
13869                        )
13870                    })
13871                })
13872                .collect()
13873        }
13874    }
13875
13876    impl Drop for MockWorkerServer {
13877        fn drop(&mut self) {
13878            self.stop.store(true, Ordering::SeqCst);
13879            let _ = TcpStream::connect(self.addr);
13880
13881            if let Some(thread) = self.thread.take() {
13882                thread.join().expect("join mock server");
13883            }
13884        }
13885    }
13886
13887    fn handle_mock_worker_request(
13888        stream: &mut TcpStream,
13889        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13890        behavior: MockWorkerBehavior,
13891    ) {
13892        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13893        let mut buffer = [0_u8; 8192];
13894        let mut request = Vec::new();
13895
13896        loop {
13897            match stream.read(&mut buffer) {
13898                Ok(0) => break,
13899                Ok(read) => {
13900                    request.extend_from_slice(&buffer[..read]);
13901                    if mock_request_is_complete(&request) {
13902                        break;
13903                    }
13904                }
13905                Err(error)
13906                    if matches!(
13907                        error.kind(),
13908                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
13909                    ) =>
13910                {
13911                    break;
13912                }
13913                Err(_) => return,
13914            }
13915        }
13916
13917        let request = String::from_utf8_lossy(&request);
13918        let body = request
13919            .split_once("\r\n\r\n")
13920            .map(|(_, body)| body)
13921            .unwrap_or_default();
13922        let path = request
13923            .lines()
13924            .next()
13925            .and_then(|line| line.split_whitespace().nth(1))
13926            .unwrap_or_default();
13927        let method = request
13928            .lines()
13929            .next()
13930            .and_then(|line| line.split_whitespace().next())
13931            .unwrap_or_default();
13932        let authorization = request.lines().find_map(|line| {
13933            let (name, value) = line.split_once(':')?;
13934            name.eq_ignore_ascii_case("Authorization")
13935                .then(|| value.trim().to_string())
13936        });
13937        let namespace = request.lines().find_map(|line| {
13938            let (name, value) = line.split_once(':')?;
13939            name.eq_ignore_ascii_case("X-Namespace")
13940                .then(|| value.trim().to_string())
13941        });
13942        let worker_protocol = request.lines().find_map(|line| {
13943            let (name, value) = line.split_once(':')?;
13944            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
13945                .then(|| value.trim().to_string())
13946        });
13947        let control_protocol = request.lines().find_map(|line| {
13948            let (name, value) = line.split_once(':')?;
13949            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
13950                .then(|| value.trim().to_string())
13951        });
13952        let request_number = {
13953            let mut requests = requests.lock().expect("captured requests");
13954            requests.push(CapturedRequest {
13955                method: method.to_string(),
13956                path: path.to_string(),
13957                authorization,
13958                namespace,
13959                worker_protocol: worker_protocol.clone(),
13960                control_protocol,
13961                body: body.to_string(),
13962                received_at: Instant::now(),
13963            });
13964            requests
13965                .iter()
13966                .filter(|request| request.path == path)
13967                .count()
13968        };
13969
13970        if behavior.reject_registration && path == "/api/worker/register" {
13971            write_mock_response(
13972                stream,
13973                "503 Service Unavailable",
13974                r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
13975            );
13976            return;
13977        }
13978
13979        if path.starts_with("/api/worker/registrations/") {
13980            if behavior.reject_deregistration_protocol {
13981                write_mock_response(
13982                    stream,
13983                    "400 Bad Request",
13984                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
13985                );
13986            } else if behavior.reject_deregistration {
13987                write_mock_response(
13988                    stream,
13989                    "403 Forbidden",
13990                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
13991                );
13992            } else {
13993                write_mock_response(
13994                    stream,
13995                    "200 OK",
13996                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
13997                );
13998            }
13999            return;
14000        }
14001
14002        let is_poll = matches!(
14003            path,
14004            "/api/worker/workflow-tasks/poll"
14005                | "/api/worker/activity-tasks/poll"
14006                | "/api/worker/query-tasks/poll"
14007        );
14008        if is_poll && request_number <= behavior.poll_failures_per_path {
14009            return;
14010        }
14011        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14012            return;
14013        }
14014        if path == "/api/worker/heartbeat"
14015            && behavior.heartbeat_failure_request == Some(request_number)
14016        {
14017            return;
14018        }
14019        if path == "/api/worker/heartbeat"
14020            && behavior.delayed_heartbeat_request == Some(request_number)
14021        {
14022            thread::sleep(behavior.heartbeat_response_delay);
14023        }
14024        if behavior.unauthorized_polls && is_poll {
14025            write_mock_response(
14026                stream,
14027                "401 Unauthorized",
14028                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14029            );
14030            return;
14031        }
14032        if behavior.draining_polls && is_poll {
14033            write_mock_response(
14034                stream,
14035                "409 Conflict",
14036                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14037            );
14038            return;
14039        }
14040
14041        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
14042            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
14043            let body = format!(
14044                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
14045            );
14046            write_mock_response(stream, "400 Bad Request", &body);
14047            return;
14048        }
14049
14050        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
14051        {
14052            write_mock_response(
14053                stream,
14054                "409 Conflict",
14055                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
14056            );
14057            return;
14058        }
14059
14060        if behavior.workflow_completion_status.is_some()
14061            && path == "/api/worker/workflow-tasks/poll"
14062            && request_number == 1
14063        {
14064            write_mock_response(
14065                stream,
14066                "200 OK",
14067                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
14068            );
14069            return;
14070        }
14071
14072        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
14073            if let (Some(status), Some(body)) = (
14074                behavior.workflow_completion_status,
14075                behavior.workflow_completion_body,
14076            ) {
14077                write_mock_response(stream, status, body);
14078                return;
14079            }
14080        }
14081
14082        if behavior.waiting_query_worker {
14083            if behavior.complete_named_signal
14084                && path == "/api/worker/workflow-tasks/poll"
14085                && request_number == 1
14086            {
14087                let body = json!({
14088                    "task": {
14089                        "task_id": "snapshot-open",
14090                        "workflow_id": "snapshot-1",
14091                        "run_id": "snapshot-run-1",
14092                        "workflow_type": "snapshot",
14093                        "payload_codec": DEFAULT_CODEC,
14094                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14095                            .expect("Avro workflow arguments"),
14096                        "history_events": [],
14097                        "workflow_task_attempt": 1,
14098                        "lease_owner": "rust-snapshot-worker"
14099                    }
14100                })
14101                .to_string();
14102                write_mock_response(stream, "200 OK", &body);
14103                return;
14104            }
14105
14106            let signal_request = request_number - usize::from(behavior.complete_named_signal);
14107            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
14108            if path == "/api/worker/workflow-tasks/poll"
14109                && signal_request >= 1
14110                && signal_request <= signal_request_limit
14111            {
14112                let finish = behavior.complete_named_signal && signal_request == 3;
14113                let amounts = if signal_request == 1 {
14114                    vec![3]
14115                } else {
14116                    vec![3, 5]
14117                };
14118                let task_id = if signal_request == 1 {
14119                    "snapshot-wait-3"
14120                } else if finish {
14121                    "snapshot-finish"
14122                } else {
14123                    "snapshot-wait-5"
14124                };
14125                let mut history_events = std::iter::once(json!({
14126                    "event_type": "SignalWaitOpened",
14127                    "payload": {"sequence": 1, "signal_name": "finish"}
14128                }))
14129                .chain(amounts.iter().enumerate().map(|(index, amount)| {
14130                    json!({
14131                        "event_type": "SignalReceived",
14132                        "payload": {
14133                            "signal_id": format!("increment-{amount}"),
14134                            "signal_name": "increment",
14135                            "workflow_sequence": index + 2,
14136                            "payload_codec": DEFAULT_CODEC,
14137                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14138                                .expect("Avro signal envelope")
14139                        }
14140                    })
14141                }))
14142                .collect::<Vec<_>>();
14143                let (resume_id, resume_name, resume_arguments) = if finish {
14144                    history_events.push(json!({
14145                        "event_type": "SignalReceived",
14146                        "payload": {
14147                            "signal_id": "finish",
14148                            "signal_name": "finish",
14149                            "workflow_sequence": 4,
14150                            "payload_codec": DEFAULT_CODEC,
14151                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14152                                .expect("Avro finish signal envelope")
14153                        }
14154                    }));
14155                    (
14156                        "finish".to_string(),
14157                        "finish".to_string(),
14158                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
14159                            .expect("Avro finish resume signal"),
14160                    )
14161                } else {
14162                    let amount = amounts.last().expect("amount");
14163                    (
14164                        format!("increment-{amount}"),
14165                        "increment".to_string(),
14166                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14167                            .expect("Avro increment resume signal"),
14168                    )
14169                };
14170                let body = json!({
14171                    "task": {
14172                        "task_id": task_id,
14173                        "workflow_id": "snapshot-1",
14174                        "run_id": "snapshot-run-1",
14175                        "workflow_type": "snapshot",
14176                        "payload_codec": DEFAULT_CODEC,
14177                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14178                            .expect("Avro workflow arguments"),
14179                        "history_events": history_events,
14180                        "workflow_task_attempt": 1,
14181                        "workflow_signal_id": resume_id,
14182                        "signal_name": resume_name,
14183                        "signal_arguments": resume_arguments,
14184                        "lease_owner": "rust-snapshot-worker"
14185                    }
14186                })
14187                .to_string();
14188                write_mock_response(stream, "200 OK", &body);
14189                return;
14190            }
14191
14192            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
14193                let history_events = [3, 5]
14194                    .into_iter()
14195                    .enumerate()
14196                    .map(|(index, amount)| {
14197                        json!({
14198                            "event_type": "SignalReceived",
14199                            "payload": {
14200                                "signal_id": format!("increment-{amount}"),
14201                                "signal_name": "increment",
14202                                "workflow_sequence": index + 2,
14203                                "payload_codec": DEFAULT_CODEC,
14204                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14205                                    .expect("Avro query signal envelope")
14206                            }
14207                        })
14208                    })
14209                    .collect::<Vec<_>>();
14210                let body = json!({
14211                    "task": {
14212                        "query_task_id": "snapshot-current",
14213                        "query_task_attempt": 1,
14214                        "lease_owner": "rust-snapshot-worker",
14215                        "workflow_id": "snapshot-1",
14216                        "run_id": "snapshot-run-1",
14217                        "workflow_type": "snapshot",
14218                        "query_name": "current",
14219                        "payload_codec": DEFAULT_CODEC,
14220                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14221                            .expect("Avro workflow arguments"),
14222                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14223                            .expect("Avro query arguments"),
14224                        "history_events": history_events,
14225                        "run_status": "waiting"
14226                    }
14227                })
14228                .to_string();
14229                write_mock_response(stream, "200 OK", &body);
14230                return;
14231            }
14232
14233            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
14234                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
14235            {
14236                write_mock_response(
14237                    stream,
14238                    "200 OK",
14239                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
14240                );
14241                return;
14242            }
14243
14244            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
14245                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
14246                return;
14247            }
14248
14249            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
14250                write_mock_response(
14251                    stream,
14252                    "200 OK",
14253                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
14254                );
14255                return;
14256            }
14257
14258            if path == "/api/worker/query-tasks/snapshot-current/complete" {
14259                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
14260                return;
14261            }
14262        }
14263
14264        if matches!(
14265            path,
14266            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
14267        ) {
14268            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14269                .expect("typed mock result");
14270            let body = json!({
14271                "result": typed_fidelity_probe().into_json().expect("result projection"),
14272                "result_envelope": result,
14273            })
14274            .to_string();
14275            write_mock_response(stream, "200 OK", &body);
14276            return;
14277        }
14278
14279        if path == "/api/workflows/typed-1" {
14280            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14281                .expect("typed mock result");
14282            let body = json!({
14283                "workflow_id": "typed-1",
14284                "run_id": "run-typed-1",
14285                "workflow_type": "typed.echo",
14286                "status": "completed",
14287                "output": typed_fidelity_probe().into_json().expect("output projection"),
14288                "output_envelope": result,
14289            })
14290            .to_string();
14291            write_mock_response(stream, "200 OK", &body);
14292            return;
14293        }
14294
14295        let (status, body) = match path {
14296            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
14297            "/api/workflows" => (
14298                "201 Created",
14299                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
14300            ),
14301            "/api/worker/register" if behavior.decline_registration => (
14302                "200 OK",
14303                r#"{"worker_id":"declined-worker","registered":false}"#,
14304            ),
14305            "/api/worker/register" if behavior.waiting_query_worker => (
14306                "200 OK",
14307                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
14308            ),
14309            "/api/worker/register" => (
14310                "200 OK",
14311                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
14312            ),
14313            "/api/worker/heartbeat" => ("200 OK", "{}"),
14314            "/api/worker/activity-tasks/poll"
14315                if behavior.cancelled_activity && request_number == 1 =>
14316            {
14317                (
14318                    "200 OK",
14319                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
14320                )
14321            }
14322            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
14323                ("200 OK", r#"{"task":null}"#)
14324            }
14325            "/api/worker/query-tasks/poll"
14326                if behavior.reject_query_completion && request_number == 1 =>
14327            {
14328                (
14329                    "200 OK",
14330                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
14331                )
14332            }
14333            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
14334            "/api/worker/query-tasks/query-capture/complete"
14335            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
14336            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
14337                "200 OK",
14338                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
14339            ),
14340            "/api/worker/activity-tasks/activity-cancel/complete" => (
14341                "409 Conflict",
14342                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
14343            ),
14344            "/api/worker/activity-tasks/activity-typed/complete"
14345            | "/api/worker/activity-tasks/activity-typed/fail"
14346            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
14347            "/api/workflows/counter-1/query/current" => (
14348                "200 OK",
14349                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
14350            ),
14351            "/api/workflows/counter-1/query/missing" => (
14352                "404 Not Found",
14353                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
14354            ),
14355            "/api/workflows/wf-lifecycle/cancel" => (
14356                "200 OK",
14357                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
14358            ),
14359            "/api/workflows/wf-lifecycle/terminate" => (
14360                "200 OK",
14361                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
14362            ),
14363            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
14364                "200 OK",
14365                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
14366            ),
14367            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
14368                "200 OK",
14369                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
14370            ),
14371            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
14372            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
14373                "409 Conflict",
14374                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
14375            ),
14376            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
14377                "200 OK",
14378                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
14379            ),
14380            "/api/workflows/wf-cancelled" => (
14381                "200 OK",
14382                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
14383            ),
14384            "/api/workflows/wf-terminated" => (
14385                "200 OK",
14386                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
14387            ),
14388            "/api/workflows/wf-timed-out" => (
14389                "200 OK",
14390                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
14391            ),
14392            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
14393                "200 OK",
14394                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
14395            ),
14396            "/api/workflows/wf-selected" => (
14397                "200 OK",
14398                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
14399            ),
14400            "/api/workflows/wf-selected/runs/run-selected" => (
14401                "200 OK",
14402                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
14403            ),
14404            _ => ("404 Not Found", r#"{"message":"not found"}"#),
14405        };
14406        write_mock_response(stream, status, body);
14407    }
14408
14409    fn mock_request_is_complete(request: &[u8]) -> bool {
14410        let Some(header_end) = request
14411            .windows(4)
14412            .position(|window| window == b"\r\n\r\n")
14413            .map(|position| position + 4)
14414        else {
14415            return false;
14416        };
14417        let headers = String::from_utf8_lossy(&request[..header_end]);
14418        let content_length = headers.lines().find_map(|line| {
14419            let (name, value) = line.split_once(':')?;
14420            name.eq_ignore_ascii_case("content-length")
14421                .then(|| value.trim().parse::<usize>().ok())
14422                .flatten()
14423        });
14424
14425        request.len() >= header_end + content_length.unwrap_or(0)
14426    }
14427
14428    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
14429        let response = format!(
14430            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
14431            body.len()
14432        );
14433
14434        let _ = stream.write_all(response.as_bytes());
14435        let _ = stream.flush();
14436    }
14437}