Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use futures_util::{future::OptionFuture, task::noop_waker_ref};
20use serde::{
21    de::DeserializeOwned,
22    ser::{SerializeMap, SerializeSeq},
23    Deserialize, Serialize, Serializer,
24};
25pub use serde_json::{json, Value};
26use thiserror::Error;
27pub use uuid::Uuid;
28
29pub const WORKER_PROTOCOL_VERSION: &str = "1.2";
30pub const CONTROL_PLANE_VERSION: &str = "2";
31pub const DEFAULT_CODEC: &str = "avro";
32pub const JSON_CODEC: &str = "json";
33pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
34/// Worker-registration capability for server-routed read-only queries.
35pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
36/// Worker-registration capability for synchronous workflow updates.
37pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
38/// First additive worker protocol that defines query-task transport.
39pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
40
41const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
42const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
43    "Workflow task waiting for scheduled history.";
44const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
45
46const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
47    "lease_expired",
48    "query_task_not_found",
49    "query_task_not_leased",
50    "query_task_timed_out",
51];
52
53/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
54pub const AVRO_VALUE_SCHEMA_JSON: &str =
55    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
56pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
57pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
58const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
59
60static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
61
62#[derive(Clone, Copy)]
63enum RequestProtocol {
64    ControlPlane,
65    Worker(&'static str),
66}
67
68pub type Result<T> = std::result::Result<T, Error>;
69
70#[derive(Debug, Error)]
71pub enum Error {
72    #[error("transport error: {0}")]
73    Transport(#[from] reqwest::Error),
74    #[error(
75        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
76    )]
77    InvalidBaseUrl,
78    #[error("json error: {0}")]
79    Json(#[from] serde_json::Error),
80    #[error("http {status}: {body}")]
81    Http {
82        status: reqwest::StatusCode,
83        body: String,
84    },
85    #[error("codec error: {0}")]
86    Codec(String),
87    #[error(transparent)]
88    QueryFailed(QueryFailure),
89    #[error(transparent)]
90    Protocol(ProtocolFailure),
91    #[error(transparent)]
92    NonDeterministicReplay(ReplayFailure),
93    #[error(transparent)]
94    ChildWorkflowFailed(ChildWorkflowFailure),
95    #[error(transparent)]
96    ActivityFailed(ActivityFailure),
97    #[error(transparent)]
98    WorkflowCommandRejected(WorkflowCommandRejection),
99    #[error(transparent)]
100    WorkflowFailed(WorkflowTerminalOutcome),
101    #[error(transparent)]
102    WorkflowCancelled(WorkflowTerminalOutcome),
103    #[error(transparent)]
104    WorkflowTerminated(WorkflowTerminalOutcome),
105    #[error(transparent)]
106    WorkflowTimedOut(WorkflowTerminalOutcome),
107    #[error(transparent)]
108    ActivityTaskRejected(ActivityTaskRejection),
109    #[error("workflow handler {0:?} is not registered")]
110    WorkflowNotRegistered(String),
111    #[error("activity handler {0:?} is not registered")]
112    ActivityNotRegistered(String),
113    #[error("workflow future yielded without emitting a durable command")]
114    WorkflowYieldedWithoutCommand,
115    #[error("workflow state lock is poisoned")]
116    WorkflowStatePoisoned,
117    #[error("timer duration is too large for the worker protocol")]
118    TimerDurationOverflow,
119    #[error("operation timed out")]
120    Timeout,
121    #[error(
122        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
123    )]
124    MissingRoleCredentials {
125        role: &'static str,
126        opposite_role: &'static str,
127    },
128    #[error("worker loop error: {0}")]
129    WorkerLoop(String),
130    #[error(
131        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
132    )]
133    UnsupportedUpdateValidators { workflow_type: String },
134    #[error("{primary}; worker deregistration also failed: {deregistration}")]
135    WorkerShutdown {
136        primary: Box<Error>,
137        deregistration: Box<Error>,
138    },
139    #[error("invalid child workflow options: {0}")]
140    InvalidChildWorkflowOptions(String),
141    #[error(transparent)]
142    InvalidActivityOptions(ActivityOptionsError),
143    #[error(transparent)]
144    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
145    #[doc(hidden)]
146    #[error("workflow requested continue as new")]
147    ContinueAsNew(ContinueAsNewRequest),
148}
149
150/// The lifecycle command sent to a workflow execution.
151#[derive(Clone, Copy, Debug, PartialEq, Eq)]
152pub enum WorkflowCommandKind {
153    Cancel,
154    Terminate,
155}
156
157impl WorkflowCommandKind {
158    fn as_str(self) -> &'static str {
159        match self {
160            Self::Cancel => "cancel",
161            Self::Terminate => "terminate",
162        }
163    }
164}
165
166/// Optional structured fields for a cancellation or termination request.
167#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
168pub struct WorkflowCommandOptions {
169    #[serde(skip_serializing_if = "Option::is_none")]
170    pub reason: Option<String>,
171    #[serde(skip_serializing_if = "Option::is_none")]
172    pub request_id: Option<String>,
173}
174
175/// Server-enforced timeout policy for a workflow start.
176///
177/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
178/// only bounds how long the caller waits. A server deadline produces a terminal
179/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
180/// `run_timeout`.
181#[derive(Clone, Debug, PartialEq, Eq)]
182pub struct WorkflowStartOptions {
183    pub execution_timeout_seconds: u64,
184    pub run_timeout_seconds: u64,
185}
186
187impl Default for WorkflowStartOptions {
188    fn default() -> Self {
189        Self {
190            execution_timeout_seconds: 3600,
191            run_timeout_seconds: 600,
192        }
193    }
194}
195
196impl WorkflowStartOptions {
197    pub fn new() -> Self {
198        Self::default()
199    }
200
201    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
202        self.execution_timeout_seconds = seconds;
203        self
204    }
205
206    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
207        self.run_timeout_seconds = seconds;
208        self
209    }
210
211    fn validate(&self) -> Result<()> {
212        if self.execution_timeout_seconds == 0 {
213            return Err(Error::Codec(
214                "execution_timeout_seconds must be at least 1".to_string(),
215            ));
216        }
217        if self.run_timeout_seconds == 0 {
218            return Err(Error::Codec(
219                "run_timeout_seconds must be at least 1".to_string(),
220            ));
221        }
222        if self.run_timeout_seconds > self.execution_timeout_seconds {
223            return Err(Error::Codec(
224                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
225            ));
226        }
227
228        Ok(())
229    }
230}
231
232/// Optional routing overrides for a continue-as-new transition.
233///
234/// Omitted values retain the current workflow type and task queue. Server-owned
235/// instance metadata is not accepted here and is carried by the server.
236#[derive(Clone, Debug, Default, PartialEq, Eq)]
237pub struct ContinueAsNewOptions {
238    pub workflow_type: Option<String>,
239    pub task_queue: Option<String>,
240}
241
242impl ContinueAsNewOptions {
243    pub fn new() -> Self {
244        Self::default()
245    }
246
247    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
248        self.workflow_type = Some(workflow_type.into());
249        self
250    }
251
252    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
253        self.task_queue = Some(task_queue.into());
254        self
255    }
256
257    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
258        for (field, value) in [
259            ("workflow_type", self.workflow_type.as_deref()),
260            ("task_queue", self.task_queue.as_deref()),
261        ] {
262            if value.is_some_and(|value| value.trim().is_empty()) {
263                return Err(ContinueAsNewOptionsError {
264                    field,
265                    message: format!("{field} must not be empty"),
266                });
267            }
268        }
269        Ok(())
270    }
271}
272
273/// A stable validation error raised before a continue-as-new command is emitted.
274#[derive(Clone, Debug, Error, PartialEq, Eq)]
275#[error("invalid continue-as-new option {field}: {message}")]
276pub struct ContinueAsNewOptionsError {
277    pub field: &'static str,
278    pub message: String,
279}
280
281/// Public history-budget information attached to the current workflow task.
282#[derive(Clone, Debug, Default, PartialEq, Eq)]
283pub struct WorkflowHistoryBudget {
284    pub event_count: u64,
285    pub size_bytes: Option<u64>,
286    pub continue_as_new_recommended: bool,
287    pub pressure: Option<String>,
288}
289
290#[doc(hidden)]
291#[derive(Clone, Debug)]
292pub struct ContinueAsNewRequest {
293    arguments: AvroValue,
294    options: ContinueAsNewOptions,
295}
296
297impl WorkflowCommandOptions {
298    pub fn new() -> Self {
299        Self::default()
300    }
301
302    pub fn reason(mut self, reason: impl Into<String>) -> Self {
303        self.reason = Some(reason.into());
304        self
305    }
306
307    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
308        self.request_id = Some(request_id.into());
309        self
310    }
311}
312
313/// The accepted, machine-readable result of a lifecycle command.
314#[derive(Clone, Debug, PartialEq)]
315pub struct WorkflowCommandResult {
316    pub command: WorkflowCommandKind,
317    pub workflow_id: String,
318    pub run_id: Option<String>,
319    pub outcome: Option<String>,
320    pub reason: Option<String>,
321    pub command_status: Option<String>,
322    pub raw: Value,
323}
324
325/// A stable rejection returned by instance- or selected-run lifecycle commands.
326#[derive(Clone, Debug, Error)]
327#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
328pub struct WorkflowCommandRejection {
329    pub command: WorkflowCommandKind,
330    pub status: u16,
331    pub reason: String,
332    pub message: String,
333    pub workflow_id: String,
334    pub run_id: Option<String>,
335    pub target_scope: Option<String>,
336    pub body: Value,
337}
338
339/// Stable terminal categories returned by [`WorkflowHandle::result`].
340#[derive(Clone, Copy, Debug, PartialEq, Eq)]
341pub enum WorkflowTerminalKind {
342    Failed,
343    Cancelled,
344    Terminated,
345    TimedOut,
346}
347
348/// A typed terminal workflow outcome with durable identity and failure metadata.
349///
350/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
351/// of parsing its display representation. Fields remain `None` when an older
352/// server did not publish that metadata.
353#[derive(Clone, Debug, Error)]
354#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
355pub struct WorkflowTerminalOutcome {
356    pub kind: WorkflowTerminalKind,
357    pub workflow_id: String,
358    pub run_id: Option<String>,
359    pub reason: String,
360    pub failure_category: Option<String>,
361    pub failure_id: Option<String>,
362    pub exception_type: Option<String>,
363    pub exception_class: Option<String>,
364    pub non_retryable: Option<bool>,
365    pub message: Option<String>,
366    pub exception: Option<Value>,
367    pub raw: Value,
368}
369
370/// A worker-side activity settlement or heartbeat rejected by durable state.
371#[derive(Clone, Debug, Error)]
372#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
373pub struct ActivityTaskRejection {
374    pub operation: String,
375    pub status: u16,
376    pub reason: String,
377    pub task_id: String,
378    pub activity_attempt_id: String,
379    pub cancel_requested: bool,
380    pub can_continue: Option<bool>,
381    pub run_closed_reason: Option<String>,
382    pub body: Value,
383}
384
385/// Stable validation categories for [`ActivityOptions`].
386#[derive(Clone, Copy, Debug, PartialEq, Eq)]
387pub enum ActivityOptionsErrorKind {
388    EmptyTaskQueue,
389    EmptyRetryPolicy,
390    InvalidMaxAttempts,
391    BackoffWithoutRetryBudget,
392    TooManyBackoffIntervals,
393    InvalidBackoffCoefficient,
394    BackoffGenerationTooLarge,
395    BackoffOverflow,
396    EmptyNonRetryableErrorType,
397    TimeoutNotPositive,
398    TimeoutOverflow,
399    TimeoutOrder,
400}
401
402/// A machine-readable activity-options validation failure.
403#[derive(Clone, Debug, Error, PartialEq, Eq)]
404#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
405pub struct ActivityOptionsError {
406    pub kind: ActivityOptionsErrorKind,
407    pub field: Option<&'static str>,
408    pub message: String,
409}
410
411impl ActivityOptionsError {
412    fn new(
413        kind: ActivityOptionsErrorKind,
414        field: Option<&'static str>,
415        message: impl Into<String>,
416    ) -> Self {
417        Self {
418            kind,
419            field,
420            message: message.into(),
421        }
422    }
423}
424
425/// Stable terminal categories returned when an awaited activity does not succeed.
426#[derive(Clone, Copy, Debug, PartialEq, Eq)]
427pub enum ActivityFailureKind {
428    Failed,
429    Cancelled,
430    TimedOut,
431}
432
433/// A stable, machine-readable terminal activity failure.
434///
435/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
436/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
437#[derive(Clone, Debug, Error)]
438#[error("activity failed ({reason}): {message}")]
439pub struct ActivityFailure {
440    pub kind: ActivityFailureKind,
441    pub reason: String,
442    pub message: String,
443    pub activity_execution_id: Option<String>,
444    pub activity_attempt_id: Option<String>,
445    pub activity_type: Option<String>,
446    pub activity_class: Option<String>,
447    pub attempt_number: Option<u64>,
448    pub failure_id: Option<String>,
449    pub failure_category: Option<String>,
450    pub timeout_kind: Option<String>,
451    pub non_retryable: bool,
452    pub exception_type: Option<String>,
453    pub exception_class: Option<String>,
454    pub code: Option<Value>,
455    pub exception: Option<Value>,
456}
457
458/// Stable terminal categories returned when an awaited child does not succeed.
459#[derive(Clone, Copy, Debug, PartialEq, Eq)]
460pub enum ChildWorkflowFailureKind {
461    Failed,
462    Cancelled,
463    Terminated,
464}
465
466/// A stable, machine-readable child workflow failure delivered to its parent.
467///
468/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
469/// of parsing the display message. Child and parent identifiers retain the
470/// relationship recorded in durable history across worker restarts.
471#[derive(Clone, Debug, Error)]
472#[error("child workflow failed ({reason}): {message}")]
473pub struct ChildWorkflowFailure {
474    pub kind: ChildWorkflowFailureKind,
475    pub reason: String,
476    pub message: String,
477    pub parent_workflow_id: Option<String>,
478    pub parent_workflow_run_id: Option<String>,
479    pub child_workflow_id: Option<String>,
480    pub child_workflow_run_id: Option<String>,
481    pub child_workflow_type: Option<String>,
482    pub failure_id: Option<String>,
483    pub failure_category: Option<String>,
484    pub exception_type: Option<String>,
485    pub exception_class: Option<String>,
486    pub non_retryable: bool,
487    pub code: Option<Value>,
488    pub exception: Option<Value>,
489}
490
491/// The identity of one durable workflow execution.
492#[derive(Clone, Debug, PartialEq, Eq)]
493pub struct WorkflowIdentity {
494    pub workflow_id: Option<String>,
495    pub run_id: Option<String>,
496}
497
498/// A successful child result together with its durable parent-child identity.
499#[derive(Clone, Debug, PartialEq)]
500pub struct ChildWorkflowResult {
501    pub parent: WorkflowIdentity,
502    pub child: WorkflowIdentity,
503    pub child_workflow_type: Option<String>,
504    pub result: Value,
505}
506
507/// Lossless successful child result for fixed Avro Value workflows.
508#[derive(Clone, Debug, PartialEq)]
509pub struct ChildWorkflowAvroResult {
510    pub parent: WorkflowIdentity,
511    pub child: WorkflowIdentity,
512    pub child_workflow_type: Option<String>,
513    pub result: AvroValue,
514}
515
516/// Server behavior when a parent closes while its child is still open.
517#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
518pub enum ParentClosePolicy {
519    #[default]
520    Abandon,
521    RequestCancel,
522    Terminate,
523}
524
525impl ParentClosePolicy {
526    fn as_str(self) -> &'static str {
527        match self {
528            Self::Abandon => "abandon",
529            Self::RequestCancel => "request_cancel",
530            Self::Terminate => "terminate",
531        }
532    }
533}
534
535/// Durable retry policy for one child workflow invocation.
536#[derive(Clone, Debug, Default, PartialEq, Eq)]
537pub struct ChildWorkflowRetryPolicy {
538    pub max_attempts: Option<u32>,
539    pub backoff_seconds: Vec<u64>,
540    pub non_retryable_error_types: Vec<String>,
541}
542
543/// Options recorded with a child-workflow command.
544///
545/// The task queue is mandatory so routing is explicit and replay-stable.
546#[derive(Clone, Debug, PartialEq, Eq)]
547pub struct ChildWorkflowOptions {
548    pub task_queue: String,
549    pub parent_close_policy: ParentClosePolicy,
550    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
551    pub execution_timeout_seconds: Option<u64>,
552    pub run_timeout_seconds: Option<u64>,
553}
554
555impl ChildWorkflowOptions {
556    pub fn new(task_queue: impl Into<String>) -> Self {
557        Self {
558            task_queue: task_queue.into(),
559            parent_close_policy: ParentClosePolicy::Abandon,
560            retry_policy: None,
561            execution_timeout_seconds: None,
562            run_timeout_seconds: None,
563        }
564    }
565
566    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
567        self.parent_close_policy = policy;
568        self
569    }
570
571    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
572        self.retry_policy = Some(policy);
573        self
574    }
575
576    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
577        self.execution_timeout_seconds = Some(seconds);
578        self
579    }
580
581    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
582        self.run_timeout_seconds = Some(seconds);
583        self
584    }
585}
586
587/// Backoff intervals for one durable activity retry policy.
588#[derive(Clone, Debug, PartialEq, Eq)]
589pub enum ActivityBackoff {
590    /// Use these intervals between attempts. The server repeats the final
591    /// interval if the retry budget contains more attempts than entries.
592    Explicit(Vec<Duration>),
593    /// Generate one interval for every retry using integer exponential growth.
594    Exponential {
595        initial_interval: Duration,
596        coefficient: u32,
597        maximum_interval: Option<Duration>,
598    },
599}
600
601/// Durable server-side retry policy for one activity execution.
602#[derive(Clone, Debug, Default, PartialEq, Eq)]
603pub struct ActivityRetryPolicy {
604    pub max_attempts: Option<u32>,
605    pub backoff: Option<ActivityBackoff>,
606    pub non_retryable_error_types: Vec<String>,
607}
608
609impl ActivityRetryPolicy {
610    /// Start a policy with a finite attempt budget, including the first attempt.
611    pub fn new(max_attempts: u32) -> Self {
612        Self {
613            max_attempts: Some(max_attempts),
614            ..Self::default()
615        }
616    }
617
618    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
619        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
620        self
621    }
622
623    pub fn exponential_backoff(
624        mut self,
625        initial_interval: Duration,
626        coefficient: u32,
627        maximum_interval: Option<Duration>,
628    ) -> Self {
629        self.backoff = Some(ActivityBackoff::Exponential {
630            initial_interval,
631            coefficient,
632            maximum_interval,
633        });
634        self
635    }
636
637    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
638        self.non_retryable_error_types.push(error_type.into());
639        self
640    }
641
642    pub fn non_retryable_error_types(
643        mut self,
644        error_types: impl IntoIterator<Item = impl Into<String>>,
645    ) -> Self {
646        self.non_retryable_error_types
647            .extend(error_types.into_iter().map(Into::into));
648        self
649    }
650}
651
652/// Options recorded atomically on one deterministic `schedule_activity` command.
653///
654/// Durations are rounded up to whole seconds when encoded, so the server never
655/// receives a shorter timeout or backoff than the caller requested.
656#[derive(Clone, Debug, Default, PartialEq, Eq)]
657pub struct ActivityOptions {
658    pub task_queue: Option<String>,
659    pub retry_policy: Option<ActivityRetryPolicy>,
660    pub start_to_close_timeout: Option<Duration>,
661    pub schedule_to_start_timeout: Option<Duration>,
662    pub schedule_to_close_timeout: Option<Duration>,
663    pub heartbeat_timeout: Option<Duration>,
664}
665
666impl ActivityOptions {
667    pub fn new() -> Self {
668        Self::default()
669    }
670
671    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
672        self.task_queue = Some(task_queue.into());
673        self
674    }
675
676    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
677        self.retry_policy = Some(policy);
678        self
679    }
680
681    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
682        self.start_to_close_timeout = Some(timeout);
683        self
684    }
685
686    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
687        self.schedule_to_start_timeout = Some(timeout);
688        self
689    }
690
691    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
692        self.schedule_to_close_timeout = Some(timeout);
693        self
694    }
695
696    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
697        self.heartbeat_timeout = Some(timeout);
698        self
699    }
700
701    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
702        if self
703            .task_queue
704            .as_deref()
705            .is_some_and(|queue| queue.trim().is_empty())
706        {
707            return Err(ActivityOptionsError::new(
708                ActivityOptionsErrorKind::EmptyTaskQueue,
709                Some("task_queue"),
710                "task_queue must not be empty",
711            ));
712        }
713
714        for (field, value) in [
715            ("start_to_close_timeout", self.start_to_close_timeout),
716            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
717            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
718            ("heartbeat_timeout", self.heartbeat_timeout),
719        ] {
720            if value.is_some_and(|value| value.is_zero()) {
721                return Err(ActivityOptionsError::new(
722                    ActivityOptionsErrorKind::TimeoutNotPositive,
723                    Some(field),
724                    format!("{field} must be positive"),
725                ));
726            }
727        }
728
729        validate_timeout_order(
730            "heartbeat_timeout",
731            self.heartbeat_timeout,
732            "start_to_close_timeout",
733            self.start_to_close_timeout,
734        )?;
735        validate_timeout_order(
736            "start_to_close_timeout",
737            self.start_to_close_timeout,
738            "schedule_to_close_timeout",
739            self.schedule_to_close_timeout,
740        )?;
741        validate_timeout_order(
742            "schedule_to_start_timeout",
743            self.schedule_to_start_timeout,
744            "schedule_to_close_timeout",
745            self.schedule_to_close_timeout,
746        )?;
747
748        Ok(ValidatedActivityOptions {
749            task_queue: self.task_queue.clone(),
750            retry_policy: self
751                .retry_policy
752                .as_ref()
753                .map(validate_activity_retry_policy)
754                .transpose()?,
755            start_to_close_timeout: timeout_seconds(
756                "start_to_close_timeout",
757                self.start_to_close_timeout,
758            )?,
759            schedule_to_start_timeout: timeout_seconds(
760                "schedule_to_start_timeout",
761                self.schedule_to_start_timeout,
762            )?,
763            schedule_to_close_timeout: timeout_seconds(
764                "schedule_to_close_timeout",
765                self.schedule_to_close_timeout,
766            )?,
767            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
768        })
769    }
770}
771
772#[derive(Clone, Debug)]
773struct ValidatedActivityOptions {
774    task_queue: Option<String>,
775    retry_policy: Option<Value>,
776    start_to_close_timeout: Option<u64>,
777    schedule_to_start_timeout: Option<u64>,
778    schedule_to_close_timeout: Option<u64>,
779    heartbeat_timeout: Option<u64>,
780}
781
782fn validate_timeout_order(
783    smaller_name: &'static str,
784    smaller: Option<Duration>,
785    larger_name: &'static str,
786    larger: Option<Duration>,
787) -> std::result::Result<(), ActivityOptionsError> {
788    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
789        return Err(ActivityOptionsError::new(
790            ActivityOptionsErrorKind::TimeoutOrder,
791            Some(smaller_name),
792            format!("{smaller_name} must be <= {larger_name}"),
793        ));
794    }
795    Ok(())
796}
797
798fn timeout_seconds(
799    field: &'static str,
800    value: Option<Duration>,
801) -> std::result::Result<Option<u64>, ActivityOptionsError> {
802    value
803        .map(|value| {
804            activity_protocol_seconds(value).ok_or_else(|| {
805                ActivityOptionsError::new(
806                    ActivityOptionsErrorKind::TimeoutOverflow,
807                    Some(field),
808                    format!("{field} is too large for the worker protocol"),
809                )
810            })
811        })
812        .transpose()
813}
814
815fn duration_seconds_ceil(value: Duration) -> Option<u64> {
816    value
817        .as_secs()
818        .checked_add(u64::from(value.subsec_nanos() > 0))
819}
820
821fn activity_protocol_seconds(value: Duration) -> Option<u64> {
822    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
823}
824
825fn validate_activity_retry_policy(
826    policy: &ActivityRetryPolicy,
827) -> std::result::Result<Value, ActivityOptionsError> {
828    if policy.max_attempts.is_none()
829        && policy.backoff.is_none()
830        && policy.non_retryable_error_types.is_empty()
831    {
832        return Err(ActivityOptionsError::new(
833            ActivityOptionsErrorKind::EmptyRetryPolicy,
834            Some("retry_policy"),
835            "retry_policy must configure at least one field",
836        ));
837    }
838    if policy.max_attempts == Some(0) {
839        return Err(ActivityOptionsError::new(
840            ActivityOptionsErrorKind::InvalidMaxAttempts,
841            Some("retry_policy.max_attempts"),
842            "max_attempts must be >= 1",
843        ));
844    }
845    if policy
846        .non_retryable_error_types
847        .iter()
848        .any(|error_type| error_type.trim().is_empty())
849    {
850        return Err(ActivityOptionsError::new(
851            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
852            Some("retry_policy.non_retryable_error_types"),
853            "non_retryable_error_types must not contain empty values",
854        ));
855    }
856
857    let backoff_seconds = match &policy.backoff {
858        None => None,
859        Some(backoff) => {
860            let max_attempts = policy.max_attempts.ok_or_else(|| {
861                ActivityOptionsError::new(
862                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
863                    Some("retry_policy.backoff"),
864                    "backoff requires max_attempts",
865                )
866            })?;
867            let retry_count = max_attempts.saturating_sub(1) as usize;
868            let intervals = match backoff {
869                ActivityBackoff::Explicit(intervals) => {
870                    if intervals.len() > retry_count {
871                        return Err(ActivityOptionsError::new(
872                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
873                            Some("retry_policy.backoff"),
874                            "backoff interval count must not exceed max_attempts - 1",
875                        ));
876                    }
877                    intervals.clone()
878                }
879                ActivityBackoff::Exponential {
880                    initial_interval,
881                    coefficient,
882                    maximum_interval,
883                } => {
884                    if *coefficient < 1 {
885                        return Err(ActivityOptionsError::new(
886                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
887                            Some("retry_policy.backoff.coefficient"),
888                            "backoff coefficient must be >= 1",
889                        ));
890                    }
891                    if retry_count > 10_000 {
892                        return Err(ActivityOptionsError::new(
893                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
894                            Some("retry_policy.max_attempts"),
895                            "generated backoff supports at most 10000 retry intervals",
896                        ));
897                    }
898                    let mut current = *initial_interval;
899                    let mut intervals = Vec::with_capacity(retry_count);
900                    for _ in 0..retry_count {
901                        let interval = maximum_interval
902                            .map(|maximum| current.min(maximum))
903                            .unwrap_or(current);
904                        intervals.push(interval);
905                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
906                            break;
907                        }
908                        current = current.checked_mul(*coefficient).ok_or_else(|| {
909                            ActivityOptionsError::new(
910                                ActivityOptionsErrorKind::BackoffOverflow,
911                                Some("retry_policy.backoff"),
912                                "generated backoff interval overflowed",
913                            )
914                        })?;
915                    }
916                    intervals
917                }
918            };
919            Some(
920                intervals
921                    .into_iter()
922                    .map(|interval| {
923                        activity_protocol_seconds(interval).ok_or_else(|| {
924                            ActivityOptionsError::new(
925                                ActivityOptionsErrorKind::BackoffOverflow,
926                                Some("retry_policy.backoff"),
927                                "backoff interval is too large for the worker protocol",
928                            )
929                        })
930                    })
931                    .collect::<std::result::Result<Vec<_>, _>>()?,
932            )
933        }
934    };
935
936    let mut encoded = serde_json::Map::new();
937    if let Some(max_attempts) = policy.max_attempts {
938        encoded.insert("max_attempts".to_string(), json!(max_attempts));
939    }
940    if let Some(backoff_seconds) = backoff_seconds {
941        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
942    }
943    if !policy.non_retryable_error_types.is_empty() {
944        let mut canonical_error_types = Vec::new();
945        for error_type in policy
946            .non_retryable_error_types
947            .iter()
948            .map(|error_type| error_type.trim())
949        {
950            if !canonical_error_types.contains(&error_type) {
951                canonical_error_types.push(error_type);
952            }
953        }
954        encoded.insert(
955            "non_retryable_error_types".to_string(),
956            json!(canonical_error_types),
957        );
958    }
959    Ok(Value::Object(encoded))
960}
961
962/// A stable, machine-readable failure raised when workflow code no longer
963/// reconstructs the durable command stream recorded in history.
964#[derive(Clone, Debug, Error)]
965#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
966pub struct ReplayFailure {
967    pub reason: String,
968    pub sequence: Option<u64>,
969    pub expected: Option<String>,
970    pub actual: Option<String>,
971    pub message: String,
972}
973
974impl ReplayFailure {
975    fn new(
976        reason: impl Into<String>,
977        sequence: Option<u64>,
978        expected: Option<String>,
979        actual: Option<String>,
980        message: impl Into<String>,
981    ) -> Self {
982        Self {
983            reason: reason.into(),
984            sequence,
985            expected,
986            actual,
987            message: message.into(),
988        }
989    }
990}
991
992/// A stable, machine-readable workflow query or query-task settlement failure.
993#[derive(Clone, Debug, Error)]
994#[error("query failed ({reason}, HTTP {status}): {message}")]
995pub struct QueryFailure {
996    pub status: u16,
997    pub reason: String,
998    pub message: String,
999    pub body: Value,
1000}
1001
1002/// A stable failure returned when a server rejects an SDK protocol version.
1003#[derive(Clone, Debug, Error)]
1004#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1005pub struct ProtocolFailure {
1006    pub status: u16,
1007    pub reason: String,
1008    pub message: String,
1009    pub supported_version: Option<String>,
1010    pub requested_version: Option<String>,
1011    pub body: Value,
1012}
1013
1014#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1015pub struct PayloadEnvelope {
1016    pub codec: String,
1017    pub blob: String,
1018}
1019
1020impl PayloadEnvelope {
1021    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1022        encode_payload(value, DEFAULT_CODEC)
1023    }
1024
1025    pub fn json<T: Serialize>(value: &T) -> Result<Self> {
1026        encode_payload(value, JSON_CODEC)
1027    }
1028
1029    /// Encode an explicit typed value, including the bytes branch that JSON
1030    /// serialization cannot represent.
1031    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1032        encode_avro_value(value)
1033    }
1034}
1035
1036/// Native adapter for the fixed language-neutral Avro Value schema.
1037#[derive(Clone, Debug, PartialEq)]
1038pub enum AvroValue {
1039    Null,
1040    Boolean(bool),
1041    Long(i64),
1042    Double(f64),
1043    Bytes(Vec<u8>),
1044    String(String),
1045    Array(Vec<AvroValue>),
1046    Map(BTreeMap<String, AvroValue>),
1047}
1048
1049impl AvroValue {
1050    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1051        Self::from_serde_value(
1052            serde_value::to_value(value).map_err(|error| {
1053                Error::Codec(format!("could not adapt value for Avro: {error}"))
1054            })?,
1055        )
1056    }
1057
1058    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1059        use serde_value::Value as SerdeValue;
1060
1061        match value {
1062            SerdeValue::Unit => Ok(Self::Null),
1063            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1064            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1065            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1066            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1067            SerdeValue::I64(value) => Ok(Self::Long(value)),
1068            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1069            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1070            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1071            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1072                Error::Codec(
1073                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1074                        .to_string(),
1075                )
1076            }),
1077            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1078            SerdeValue::F64(value) => Self::finite_double(value),
1079            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1080            SerdeValue::String(value) => Ok(Self::String(value)),
1081            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1082            SerdeValue::Option(None) => Ok(Self::Null),
1083            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1084                Self::from_serde_value(*value)
1085            }
1086            SerdeValue::Seq(values) => values
1087                .into_iter()
1088                .map(Self::from_serde_value)
1089                .collect::<Result<Vec<_>>>()
1090                .map(Self::Array),
1091            SerdeValue::Map(values) => values
1092                .into_iter()
1093                .map(|(key, value)| {
1094                    let SerdeValue::String(key) = key else {
1095                        return Err(Error::Codec(
1096                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1097                        ));
1098                    };
1099
1100                    Ok((key, Self::from_serde_value(value)?))
1101                })
1102                .collect::<Result<BTreeMap<_, _>>>()
1103                .map(Self::Map),
1104        }
1105    }
1106
1107    fn finite_double(value: f64) -> Result<Self> {
1108        if !value.is_finite() {
1109            return Err(Error::Codec(
1110                "non_finite_float: Avro Value doubles must be finite".to_string(),
1111            ));
1112        }
1113
1114        Ok(Self::Double(value))
1115    }
1116
1117    fn into_json(self) -> Result<Value> {
1118        match self {
1119            Self::Null => Ok(Value::Null),
1120            Self::Boolean(value) => Ok(Value::Bool(value)),
1121            Self::Long(value) => Ok(Value::Number(value.into())),
1122            Self::Double(value) => serde_json::Number::from_f64(value)
1123                .map(Value::Number)
1124                .ok_or_else(|| {
1125                    Error::Codec(
1126                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1127                    )
1128                }),
1129            Self::Bytes(value) => Ok(json!({
1130                "$type": "bytes",
1131                "base64": BASE64.encode(value),
1132            })),
1133            Self::String(value) => Ok(Value::String(value)),
1134            Self::Array(values) => values
1135                .into_iter()
1136                .map(Self::into_json)
1137                .collect::<Result<Vec<_>>>()
1138                .map(Value::Array),
1139            Self::Map(values) => values
1140                .into_iter()
1141                .map(|(key, value)| Ok((key, value.into_json()?)))
1142                .collect::<Result<serde_json::Map<_, _>>>()
1143                .map(Value::Object),
1144        }
1145    }
1146
1147    fn into_serde_value(self) -> serde_value::Value {
1148        use serde_value::Value as SerdeValue;
1149
1150        match self {
1151            Self::Null => SerdeValue::Unit,
1152            Self::Boolean(value) => SerdeValue::Bool(value),
1153            Self::Long(value) => SerdeValue::I64(value),
1154            Self::Double(value) => SerdeValue::F64(value),
1155            Self::Bytes(value) => SerdeValue::Bytes(value),
1156            Self::String(value) => SerdeValue::String(value),
1157            Self::Array(values) => {
1158                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1159            }
1160            Self::Map(values) => SerdeValue::Map(
1161                values
1162                    .into_iter()
1163                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1164                    .collect(),
1165            ),
1166        }
1167    }
1168
1169    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1170        self.into_serde_value().deserialize_into().map_err(|error| {
1171            Error::Codec(format!(
1172                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1173            ))
1174        })
1175    }
1176}
1177
1178impl Serialize for AvroValue {
1179    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1180    where
1181        S: Serializer,
1182    {
1183        match self {
1184            Self::Null => serializer.serialize_unit(),
1185            Self::Boolean(value) => serializer.serialize_bool(*value),
1186            Self::Long(value) => serializer.serialize_i64(*value),
1187            Self::Double(value) => serializer.serialize_f64(*value),
1188            Self::Bytes(value) => serializer.serialize_bytes(value),
1189            Self::String(value) => serializer.serialize_str(value),
1190            Self::Array(values) => {
1191                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1192                for value in values {
1193                    sequence.serialize_element(value)?;
1194                }
1195                sequence.end()
1196            }
1197            Self::Map(values) => {
1198                let mut map = serializer.serialize_map(Some(values.len()))?;
1199                for (key, value) in values {
1200                    map.serialize_entry(key, value)?;
1201                }
1202                map.end()
1203            }
1204        }
1205    }
1206}
1207
1208pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1209    let datum = avro_value_to_datum(value)?;
1210    let datum = to_avro_datum(avro_value_schema()?, datum)
1211        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
1212    let mut bytes = Vec::with_capacity(datum.len() + 10);
1213    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1214    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1215    bytes.extend_from_slice(&datum);
1216    Ok(PayloadEnvelope {
1217        codec: DEFAULT_CODEC.to_string(),
1218        blob: BASE64.encode(bytes),
1219    })
1220}
1221
1222pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1223    if envelope.codec != DEFAULT_CODEC {
1224        return Err(Error::Codec(format!(
1225            "unsupported payload codec {:?}",
1226            envelope.codec
1227        )));
1228    }
1229    decode_avro_value_blob(&envelope.blob)
1230}
1231
1232pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1233    let blob = match codec {
1234        JSON_CODEC => serde_json::to_string(value)?,
1235        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1236        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1237    };
1238
1239    Ok(PayloadEnvelope {
1240        codec: codec.to_string(),
1241        blob,
1242    })
1243}
1244
1245pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1246    match envelope.codec.as_str() {
1247        JSON_CODEC => Ok(serde_json::from_str(&envelope.blob)?),
1248        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1249        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1250    }
1251}
1252
1253#[cfg(test)]
1254fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1255    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1256}
1257
1258fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1259    if value.is_null() {
1260        return Ok(Value::Null);
1261    }
1262
1263    if let Some(object) = value.as_object() {
1264        if let (Some(codec), Some(blob)) = (
1265            object.get("codec").and_then(Value::as_str),
1266            object.get("blob").and_then(Value::as_str),
1267        ) {
1268            return decode_blob(blob, codec);
1269        }
1270    }
1271
1272    if let Some(blob) = value.as_str() {
1273        return decode_blob(blob, fallback_codec);
1274    }
1275
1276    Ok(value.clone())
1277}
1278
1279fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
1280    let envelope = match codec {
1281        DEFAULT_CODEC => encode_avro_value(value)?,
1282        JSON_CODEC => PayloadEnvelope {
1283            codec: JSON_CODEC.to_string(),
1284            blob: serde_json::to_string(&value.clone().into_json()?)?,
1285        },
1286        other => return Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1287    };
1288    Ok(serde_json::to_value(envelope)?)
1289}
1290
1291fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
1292    if value.is_null() {
1293        return Ok(AvroValue::Null);
1294    }
1295
1296    if let Some(object) = value.as_object() {
1297        if let (Some(codec), Some(blob)) = (
1298            object.get("codec").and_then(Value::as_str),
1299            object.get("blob").and_then(Value::as_str),
1300        ) {
1301            return match codec {
1302                DEFAULT_CODEC => decode_avro_value_blob(blob),
1303                JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1304                other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1305            };
1306        }
1307    }
1308
1309    if let Some(blob) = value.as_str() {
1310        return match fallback_codec {
1311            DEFAULT_CODEC => decode_avro_value_blob(blob),
1312            JSON_CODEC => AvroValue::from_serialize(&serde_json::from_str::<Value>(blob)?),
1313            other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1314        };
1315    }
1316
1317    AvroValue::from_serialize(value)
1318}
1319
1320fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
1321    match value {
1322        AvroValue::Null => AvroValue::Array(Vec::new()),
1323        AvroValue::Array(_) => value,
1324        other => AvroValue::Array(vec![other]),
1325    }
1326}
1327
1328fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
1329    match codec {
1330        JSON_CODEC => Ok(serde_json::from_str(blob)?),
1331        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
1332        other => Err(Error::Codec(format!("unsupported payload codec {other:?}"))),
1333    }
1334}
1335
1336fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
1337    let bytes = BASE64.decode(blob).map_err(|err| {
1338        Error::Codec(format!(
1339            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
1340        ))
1341    })?;
1342
1343    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
1344        return Err(Error::Codec(
1345            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
1346        ));
1347    }
1348
1349    let fingerprint: [u8; 8] = bytes[2..10]
1350        .try_into()
1351        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
1352    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
1353        return Err(Error::Codec(format!(
1354            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
1355            fingerprint
1356                .iter()
1357                .map(|byte| format!("{byte:02x}"))
1358                .collect::<String>()
1359        )));
1360    }
1361
1362    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
1363    // The current fingerprint selects the current immutable schema, so reader
1364    // resolution would only re-walk the same recursive union. Future retained
1365    // writer fingerprints supply a distinct reader schema in this branch.
1366    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
1367    if datum_reader.truncated {
1368        return Err(Error::Codec(
1369            "invalid_payload_framing: truncated Avro Value datum".to_string(),
1370        ));
1371    }
1372    let datum = datum.map_err(|err| {
1373        Error::Codec(format!(
1374            "invalid_payload_framing: malformed Avro Value datum: {err}"
1375        ))
1376    })?;
1377    if datum_reader.remaining() != 0 {
1378        return Err(Error::Codec(format!(
1379            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
1380            datum_reader.remaining()
1381        )));
1382    }
1383    avro_value_from_datum(datum)
1384}
1385
1386struct StrictAvroDatumReader<'a> {
1387    bytes: &'a [u8],
1388    offset: usize,
1389    truncated: bool,
1390}
1391
1392impl<'a> StrictAvroDatumReader<'a> {
1393    fn new(bytes: &'a [u8]) -> Self {
1394        Self {
1395            bytes,
1396            offset: 0,
1397            truncated: false,
1398        }
1399    }
1400
1401    fn remaining(&self) -> usize {
1402        self.bytes.len() - self.offset
1403    }
1404}
1405
1406impl Read for StrictAvroDatumReader<'_> {
1407    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
1408        let count = buffer.len().min(self.remaining());
1409        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
1410        self.offset += count;
1411        if count < buffer.len() {
1412            self.truncated = true;
1413        }
1414
1415        Ok(count)
1416    }
1417}
1418
1419fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
1420    let branch = match value {
1421        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
1422        AvroValue::Boolean(value) => AvroDatum::Union(
1423            1,
1424            Box::new(AvroDatum::Record(vec![(
1425                "boolean".to_string(),
1426                AvroDatum::Boolean(*value),
1427            )])),
1428        ),
1429        AvroValue::Long(value) => AvroDatum::Union(
1430            2,
1431            Box::new(AvroDatum::Record(vec![(
1432                "long".to_string(),
1433                AvroDatum::Long(*value),
1434            )])),
1435        ),
1436        AvroValue::Double(value) => {
1437            if !value.is_finite() {
1438                return Err(Error::Codec(
1439                    "non_finite_float: Avro Value doubles must be finite".to_string(),
1440                ));
1441            }
1442            AvroDatum::Union(
1443                3,
1444                Box::new(AvroDatum::Record(vec![(
1445                    "double".to_string(),
1446                    AvroDatum::Double(*value),
1447                )])),
1448            )
1449        }
1450        AvroValue::Bytes(value) => AvroDatum::Union(
1451            4,
1452            Box::new(AvroDatum::Record(vec![(
1453                "bytes".to_string(),
1454                AvroDatum::Bytes(value.clone()),
1455            )])),
1456        ),
1457        AvroValue::String(value) => AvroDatum::Union(
1458            5,
1459            Box::new(AvroDatum::Record(vec![(
1460                "string".to_string(),
1461                AvroDatum::String(value.clone()),
1462            )])),
1463        ),
1464        AvroValue::Array(values) => AvroDatum::Union(
1465            6,
1466            Box::new(AvroDatum::Record(vec![(
1467                "items".to_string(),
1468                AvroDatum::Array(
1469                    values
1470                        .iter()
1471                        .map(avro_value_to_datum)
1472                        .collect::<Result<Vec<_>>>()?,
1473                ),
1474            )])),
1475        ),
1476        AvroValue::Map(values) => AvroDatum::Union(
1477            7,
1478            Box::new(AvroDatum::Record(vec![(
1479                "entries".to_string(),
1480                AvroDatum::Map(
1481                    values
1482                        .iter()
1483                        .map(|(key, value)| Ok((key.clone(), avro_value_to_datum(value)?)))
1484                        .collect::<Result<HashMap<_, _>>>()?,
1485                ),
1486            )])),
1487        ),
1488    };
1489    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
1490}
1491
1492fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
1493    let AvroDatum::Record(mut outer) = datum else {
1494        return Err(Error::Codec(
1495            "invalid_payload_framing: datum is not a Value record".to_string(),
1496        ));
1497    };
1498    let (_, branch) = outer
1499        .pop()
1500        .filter(|(name, _)| name == "value")
1501        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
1502    let AvroDatum::Union(_, branch) = branch else {
1503        return Err(Error::Codec(
1504            "invalid_payload_framing: invalid Value union".to_string(),
1505        ));
1506    };
1507    match *branch {
1508        AvroDatum::Null => Ok(AvroValue::Null),
1509        AvroDatum::Record(mut fields) => {
1510            let (name, value) = fields.pop().ok_or_else(|| {
1511                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
1512            })?;
1513            match (name.as_str(), value) {
1514                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
1515                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
1516                ("double", AvroDatum::Double(value)) if value.is_finite() => {
1517                    Ok(AvroValue::Double(value))
1518                }
1519                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
1520                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
1521                ("items", AvroDatum::Array(values)) => values
1522                    .into_iter()
1523                    .map(avro_value_from_datum)
1524                    .collect::<Result<Vec<_>>>()
1525                    .map(AvroValue::Array),
1526                ("entries", AvroDatum::Map(values)) => values
1527                    .into_iter()
1528                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
1529                    .collect::<Result<BTreeMap<_, _>>>()
1530                    .map(AvroValue::Map),
1531                _ => Err(Error::Codec(
1532                    "invalid_payload_framing: unknown Value branch".to_string(),
1533                )),
1534            }
1535        }
1536        _ => Err(Error::Codec(
1537            "invalid_payload_framing: invalid Value branch".to_string(),
1538        )),
1539    }
1540}
1541
1542fn avro_value_schema() -> Result<&'static Schema> {
1543    match AVRO_VALUE_SCHEMA.get_or_init(|| {
1544        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
1545            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
1546    }) {
1547        Ok(schema) => Ok(schema),
1548        Err(message) => Err(Error::Codec(message.clone())),
1549    }
1550}
1551
1552#[derive(Clone, Debug)]
1553pub struct Client {
1554    http: reqwest::Client,
1555    base_url: String,
1556    token: Option<String>,
1557    control_token: Option<String>,
1558    worker_token: Option<String>,
1559    namespace: String,
1560}
1561
1562impl Client {
1563    pub fn new(base_url: impl Into<String>) -> Result<Self> {
1564        Self::builder(base_url).build()
1565    }
1566
1567    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
1568        ClientBuilder {
1569            base_url: base_url.into(),
1570            token: None,
1571            control_token: None,
1572            worker_token: None,
1573            namespace: "default".to_string(),
1574            timeout: Duration::from_secs(60),
1575        }
1576    }
1577
1578    pub async fn health(&self) -> Result<Value> {
1579        self.request_json(
1580            reqwest::Method::GET,
1581            "/health",
1582            RequestProtocol::ControlPlane,
1583            Option::<&Value>::None,
1584        )
1585        .await
1586    }
1587
1588    pub async fn cluster_info(&self) -> Result<Value> {
1589        self.request_json(
1590            reqwest::Method::GET,
1591            "/cluster/info",
1592            RequestProtocol::ControlPlane,
1593            Option::<&Value>::None,
1594        )
1595        .await
1596    }
1597
1598    pub async fn start_workflow<T: Serialize>(
1599        &self,
1600        workflow_type: &str,
1601        task_queue: &str,
1602        workflow_id: &str,
1603        input: T,
1604    ) -> Result<WorkflowHandle> {
1605        self.start_workflow_with_options(
1606            workflow_type,
1607            task_queue,
1608            workflow_id,
1609            WorkflowStartOptions::default(),
1610            input,
1611        )
1612        .await
1613    }
1614
1615    /// Start a workflow with explicit server-enforced execution and run
1616    /// deadlines.
1617    pub async fn start_workflow_with_options<T: Serialize>(
1618        &self,
1619        workflow_type: &str,
1620        task_queue: &str,
1621        workflow_id: &str,
1622        options: WorkflowStartOptions,
1623        input: T,
1624    ) -> Result<WorkflowHandle> {
1625        options.validate()?;
1626        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1627        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1628        let body = json!({
1629            "workflow_id": workflow_id,
1630            "workflow_type": workflow_type,
1631            "task_queue": task_queue,
1632            "input": input_envelope,
1633            "execution_timeout_seconds": options.execution_timeout_seconds,
1634            "run_timeout_seconds": options.run_timeout_seconds
1635        });
1636
1637        let data: Value = self
1638            .request_json(
1639                reqwest::Method::POST,
1640                "/workflows",
1641                RequestProtocol::ControlPlane,
1642                Some(&body),
1643            )
1644            .await?;
1645
1646        Ok(WorkflowHandle {
1647            client: self.clone(),
1648            workflow_id: data
1649                .get("workflow_id")
1650                .and_then(Value::as_str)
1651                .unwrap_or(workflow_id)
1652                .to_string(),
1653            run_id: data
1654                .get("run_id")
1655                .and_then(Value::as_str)
1656                .map(str::to_string),
1657            workflow_type: data
1658                .get("workflow_type")
1659                .and_then(Value::as_str)
1660                .unwrap_or(workflow_type)
1661                .to_string(),
1662        })
1663    }
1664
1665    pub async fn signal_workflow<T: Serialize>(
1666        &self,
1667        workflow_id: &str,
1668        signal_name: &str,
1669        input: T,
1670    ) -> Result<Value> {
1671        self.signal_workflow_target(workflow_id, None, signal_name, input)
1672            .await
1673    }
1674
1675    /// Signal only if `run_id` is still the current run for this instance.
1676    pub async fn signal_workflow_run<T: Serialize>(
1677        &self,
1678        workflow_id: &str,
1679        run_id: &str,
1680        signal_name: &str,
1681        input: T,
1682    ) -> Result<Value> {
1683        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
1684            .await
1685    }
1686
1687    async fn signal_workflow_target<T: Serialize>(
1688        &self,
1689        workflow_id: &str,
1690        run_id: Option<&str>,
1691        signal_name: &str,
1692        input: T,
1693    ) -> Result<Value> {
1694        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1695        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1696        let body = json!({
1697            "input": input_envelope
1698        });
1699        let path = match run_id {
1700            Some(run_id) => {
1701                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
1702            }
1703            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
1704        };
1705        self.request_json(
1706            reqwest::Method::POST,
1707            &path,
1708            RequestProtocol::ControlPlane,
1709            Some(&body),
1710        )
1711        .await
1712    }
1713
1714    /// Request cooperative cancellation of the current run for an instance.
1715    pub async fn cancel_workflow(
1716        &self,
1717        workflow_id: &str,
1718        options: WorkflowCommandOptions,
1719    ) -> Result<WorkflowCommandResult> {
1720        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
1721            .await
1722    }
1723
1724    /// Request cooperative cancellation only if `run_id` is still current.
1725    pub async fn cancel_workflow_run(
1726        &self,
1727        workflow_id: &str,
1728        run_id: &str,
1729        options: WorkflowCommandOptions,
1730    ) -> Result<WorkflowCommandResult> {
1731        self.workflow_command(
1732            workflow_id,
1733            Some(run_id),
1734            WorkflowCommandKind::Cancel,
1735            options,
1736        )
1737        .await
1738    }
1739
1740    /// Forcefully terminate the current run for an instance.
1741    pub async fn terminate_workflow(
1742        &self,
1743        workflow_id: &str,
1744        options: WorkflowCommandOptions,
1745    ) -> Result<WorkflowCommandResult> {
1746        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
1747            .await
1748    }
1749
1750    /// Forcefully terminate only if `run_id` is still current.
1751    pub async fn terminate_workflow_run(
1752        &self,
1753        workflow_id: &str,
1754        run_id: &str,
1755        options: WorkflowCommandOptions,
1756    ) -> Result<WorkflowCommandResult> {
1757        self.workflow_command(
1758            workflow_id,
1759            Some(run_id),
1760            WorkflowCommandKind::Terminate,
1761            options,
1762        )
1763        .await
1764    }
1765
1766    async fn workflow_command(
1767        &self,
1768        workflow_id: &str,
1769        run_id: Option<&str>,
1770        command: WorkflowCommandKind,
1771        options: WorkflowCommandOptions,
1772    ) -> Result<WorkflowCommandResult> {
1773        let path = match run_id {
1774            Some(run_id) => format!(
1775                "/workflows/{workflow_id}/runs/{run_id}/{}",
1776                command.as_str()
1777            ),
1778            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
1779        };
1780        let data = match self
1781            .request_json(
1782                reqwest::Method::POST,
1783                &path,
1784                RequestProtocol::ControlPlane,
1785                Some(&options),
1786            )
1787            .await
1788        {
1789            Ok(data) => data,
1790            Err(Error::Http { status, body }) => {
1791                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
1792                    command,
1793                    status,
1794                    body,
1795                    workflow_id,
1796                    run_id,
1797                )));
1798            }
1799            Err(error) => return Err(error),
1800        };
1801
1802        Ok(workflow_command_result(command, data, workflow_id, run_id))
1803    }
1804
1805    /// Execute a named, read-only query against a running or completed workflow.
1806    ///
1807    /// Arguments and results use the platform payload envelope. Server and
1808    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
1809    /// reason, HTTP status, and original response body.
1810    pub async fn query_workflow<T: Serialize>(
1811        &self,
1812        workflow_id: &str,
1813        query_name: &str,
1814        input: T,
1815    ) -> Result<Value> {
1816        self.query_workflow_target(workflow_id, None, query_name, input)
1817            .await
1818    }
1819
1820    /// Query only if `run_id` is still current, preventing accidental retargeting.
1821    pub async fn query_workflow_run<T: Serialize>(
1822        &self,
1823        workflow_id: &str,
1824        run_id: &str,
1825        query_name: &str,
1826        input: T,
1827    ) -> Result<Value> {
1828        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
1829            .await
1830    }
1831
1832    /// Query a workflow and return the lossless fixed Avro Value result.
1833    pub async fn query_workflow_avro_value<T: Serialize>(
1834        &self,
1835        workflow_id: &str,
1836        query_name: &str,
1837        input: T,
1838    ) -> Result<AvroValue> {
1839        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
1840            .await
1841    }
1842
1843    /// Query a selected run and return the lossless fixed Avro Value result.
1844    pub async fn query_workflow_run_avro_value<T: Serialize>(
1845        &self,
1846        workflow_id: &str,
1847        run_id: &str,
1848        query_name: &str,
1849        input: T,
1850    ) -> Result<AvroValue> {
1851        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
1852            .await
1853    }
1854
1855    async fn query_workflow_avro_value_target<T: Serialize>(
1856        &self,
1857        workflow_id: &str,
1858        run_id: Option<&str>,
1859        query_name: &str,
1860        input: T,
1861    ) -> Result<AvroValue> {
1862        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1863        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
1864        let path = match run_id {
1865            Some(run_id) => {
1866                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1867            }
1868            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1869        };
1870        let response: Value = match self
1871            .request_json(
1872                reqwest::Method::POST,
1873                &path,
1874                RequestProtocol::ControlPlane,
1875                Some(&body),
1876            )
1877            .await
1878        {
1879            Ok(response) => response,
1880            Err(Error::Http { status, body }) => {
1881                return Err(Error::QueryFailed(query_failure(status, body)));
1882            }
1883            Err(error) => return Err(error),
1884        };
1885
1886        let envelope = response
1887            .get("result_envelope")
1888            .filter(|envelope| !envelope.is_null())
1889            .ok_or_else(|| {
1890                Error::Codec(
1891                    "missing_payload_envelope: typed query result requires result_envelope"
1892                        .to_string(),
1893                )
1894            })?;
1895        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1896    }
1897
1898    async fn query_workflow_target<T: Serialize>(
1899        &self,
1900        workflow_id: &str,
1901        run_id: Option<&str>,
1902        query_name: &str,
1903        input: T,
1904    ) -> Result<Value> {
1905        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1906        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
1907        let body = json!({
1908            "input": input_envelope
1909        });
1910        let path = match run_id {
1911            Some(run_id) => {
1912                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
1913            }
1914            None => format!("/workflows/{workflow_id}/query/{query_name}"),
1915        };
1916        let response: Value = match self
1917            .request_json(
1918                reqwest::Method::POST,
1919                &path,
1920                RequestProtocol::ControlPlane,
1921                Some(&body),
1922            )
1923            .await
1924        {
1925            Ok(response) => response,
1926            Err(Error::Http { status, body }) => {
1927                return Err(Error::QueryFailed(query_failure(status, body)));
1928            }
1929            Err(error) => return Err(error),
1930        };
1931
1932        if let Some(envelope) = response
1933            .get("result_envelope")
1934            .filter(|envelope| !envelope.is_null())
1935        {
1936            return decode_wire_value(envelope, DEFAULT_CODEC);
1937        }
1938
1939        Ok(response.get("result").cloned().unwrap_or(Value::Null))
1940    }
1941
1942    /// Send a synchronous update using fixed Avro Value arguments.
1943    pub async fn update_workflow<T: Serialize>(
1944        &self,
1945        workflow_id: &str,
1946        update_name: &str,
1947        input: T,
1948        request_id: Option<&str>,
1949    ) -> Result<Value> {
1950        let response = self
1951            .update_workflow_response(workflow_id, update_name, input, request_id)
1952            .await?;
1953        if let Some(envelope) = response
1954            .get("result_envelope")
1955            .filter(|envelope| !envelope.is_null())
1956        {
1957            return decode_wire_value(envelope, DEFAULT_CODEC);
1958        }
1959        Ok(response.get("result").cloned().unwrap_or(response))
1960    }
1961
1962    /// Send a synchronous update and retain a bytes-capable Avro result.
1963    pub async fn update_workflow_avro_value<T: Serialize>(
1964        &self,
1965        workflow_id: &str,
1966        update_name: &str,
1967        input: T,
1968        request_id: Option<&str>,
1969    ) -> Result<AvroValue> {
1970        let response = self
1971            .update_workflow_response(workflow_id, update_name, input, request_id)
1972            .await?;
1973        let envelope = response
1974            .get("result_envelope")
1975            .filter(|envelope| !envelope.is_null())
1976            .ok_or_else(|| {
1977                Error::Codec(
1978                    "missing_payload_envelope: typed update result requires result_envelope"
1979                        .to_string(),
1980                )
1981            })?;
1982        decode_wire_avro_value(envelope, DEFAULT_CODEC)
1983    }
1984
1985    async fn update_workflow_response<T: Serialize>(
1986        &self,
1987        workflow_id: &str,
1988        update_name: &str,
1989        input: T,
1990        request_id: Option<&str>,
1991    ) -> Result<Value> {
1992        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
1993        let mut body = json!({
1994            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
1995            "wait_for": "completed",
1996        });
1997        if let Some(request_id) = request_id {
1998            body["request_id"] = json!(request_id);
1999        }
2000        self.request_json(
2001            reqwest::Method::POST,
2002            &format!("/workflows/{workflow_id}/update/{update_name}"),
2003            RequestProtocol::ControlPlane,
2004            Some(&body),
2005        )
2006        .await
2007    }
2008
2009    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2010        let path = format!("/workflows/{workflow_id}");
2011        let mut data: WorkflowDescription = self
2012            .request_json(
2013                reqwest::Method::GET,
2014                &path,
2015                RequestProtocol::ControlPlane,
2016                Option::<&Value>::None,
2017            )
2018            .await?;
2019        data.decode_payloads()?;
2020        Ok(data)
2021    }
2022
2023    /// Describe one selected run, including historical terminal runs.
2024    pub async fn describe_workflow_run(
2025        &self,
2026        workflow_id: &str,
2027        run_id: &str,
2028    ) -> Result<WorkflowDescription> {
2029        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2030        let mut data: WorkflowDescription = self
2031            .request_json(
2032                reqwest::Method::GET,
2033                &path,
2034                RequestProtocol::ControlPlane,
2035                Option::<&Value>::None,
2036            )
2037            .await?;
2038        data.decode_payloads()?;
2039        Ok(data)
2040    }
2041
2042    pub async fn register_worker(
2043        &self,
2044        worker_id: &str,
2045        task_queue: &str,
2046        supported_workflow_types: Vec<String>,
2047        supported_activity_types: Vec<String>,
2048        max_concurrent_workflow_tasks: usize,
2049        max_concurrent_activity_tasks: usize,
2050    ) -> Result<RegisterWorkerResponse> {
2051        self.register_worker_with_capabilities(
2052            worker_id,
2053            task_queue,
2054            supported_workflow_types,
2055            supported_activity_types,
2056            max_concurrent_workflow_tasks,
2057            max_concurrent_activity_tasks,
2058            Vec::new(),
2059        )
2060        .await
2061    }
2062
2063    /// Register a worker and explicitly advertise additive worker capabilities.
2064    pub async fn register_worker_with_capabilities(
2065        &self,
2066        worker_id: &str,
2067        task_queue: &str,
2068        supported_workflow_types: Vec<String>,
2069        supported_activity_types: Vec<String>,
2070        max_concurrent_workflow_tasks: usize,
2071        max_concurrent_activity_tasks: usize,
2072        capabilities: Vec<String>,
2073    ) -> Result<RegisterWorkerResponse> {
2074        self.register_worker_with_command_contracts(
2075            worker_id,
2076            task_queue,
2077            supported_workflow_types,
2078            supported_activity_types,
2079            max_concurrent_workflow_tasks,
2080            max_concurrent_activity_tasks,
2081            capabilities,
2082            Value::Object(serde_json::Map::new()),
2083        )
2084        .await
2085    }
2086
2087    /// Register a worker and advertise its named query and update handlers.
2088    ///
2089    /// This Rust SDK cannot execute synchronous pre-accept update validation,
2090    /// so a workflow contract with a non-empty or malformed
2091    /// `update_validators` declaration returns
2092    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
2093    #[allow(clippy::too_many_arguments)]
2094    pub async fn register_worker_with_command_contracts(
2095        &self,
2096        worker_id: &str,
2097        task_queue: &str,
2098        supported_workflow_types: Vec<String>,
2099        supported_activity_types: Vec<String>,
2100        max_concurrent_workflow_tasks: usize,
2101        max_concurrent_activity_tasks: usize,
2102        capabilities: Vec<String>,
2103        workflow_command_contracts: Value,
2104    ) -> Result<RegisterWorkerResponse> {
2105        if let Some(contracts) = workflow_command_contracts.as_object() {
2106            for (workflow_type, contract) in contracts {
2107                let Some(update_validators) = contract.get("update_validators") else {
2108                    continue;
2109                };
2110                if !update_validators
2111                    .as_array()
2112                    .is_some_and(|validators| validators.is_empty())
2113                {
2114                    return Err(Error::UnsupportedUpdateValidators {
2115                        workflow_type: workflow_type.clone(),
2116                    });
2117                }
2118            }
2119        }
2120
2121        let mut body = json!({
2122            "worker_id": worker_id,
2123            "task_queue": task_queue,
2124            "runtime": "rust",
2125            "sdk_version": SDK_VERSION,
2126            "supported_workflow_types": supported_workflow_types,
2127            "supported_activity_types": supported_activity_types,
2128            "capabilities": capabilities,
2129            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
2130            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
2131        });
2132        if workflow_command_contracts
2133            .as_object()
2134            .is_some_and(|contracts| !contracts.is_empty())
2135        {
2136            body["workflow_command_contracts"] = workflow_command_contracts;
2137        }
2138
2139        self.request_json(
2140            reqwest::Method::POST,
2141            "/worker/register",
2142            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2143            Some(&body),
2144        )
2145        .await
2146    }
2147
2148    /// Gracefully remove one worker's registration through the worker plane.
2149    ///
2150    /// This operation is separate from operator-facing worker management. It
2151    /// uses worker-protocol authentication and returns the server's lease
2152    /// recovery result.
2153    pub async fn deregister_worker_registration(
2154        &self,
2155        worker_id: &str,
2156    ) -> Result<WorkerDeregistrationEnvelope> {
2157        let path = format!(
2158            "/worker/registrations/{}",
2159            percent_encode_path_segment(worker_id)
2160        );
2161        self.request_json(
2162            reqwest::Method::DELETE,
2163            &path,
2164            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2165            Option::<&Value>::None,
2166        )
2167        .await
2168    }
2169
2170    /// Long-poll for an ephemeral, read-only workflow query task.
2171    pub async fn poll_query_task(
2172        &self,
2173        worker_id: &str,
2174        task_queue: &str,
2175        timeout: Duration,
2176    ) -> Result<Option<QueryTask>> {
2177        Ok(self
2178            .poll_query_task_response(worker_id, task_queue, timeout)
2179            .await?
2180            .task)
2181    }
2182
2183    /// Poll a query task while preserving server stop and drain metadata.
2184    pub async fn poll_query_task_response(
2185        &self,
2186        worker_id: &str,
2187        task_queue: &str,
2188        timeout: Duration,
2189    ) -> Result<PollQueryTaskResponse> {
2190        let poll_request_id = unique_request_id("rust-query-poll");
2191        self.poll_query_task_response_with_request_id(
2192            worker_id,
2193            task_queue,
2194            timeout,
2195            &poll_request_id,
2196            1,
2197        )
2198        .await
2199    }
2200
2201    async fn poll_query_task_response_with_request_id(
2202        &self,
2203        worker_id: &str,
2204        task_queue: &str,
2205        timeout: Duration,
2206        poll_request_id: &str,
2207        transport_retries: usize,
2208    ) -> Result<PollQueryTaskResponse> {
2209        let timeout_seconds = long_poll_timeout_seconds(timeout);
2210        let body = json!({
2211            "worker_id": worker_id,
2212            "task_queue": task_queue,
2213            "poll_request_id": poll_request_id,
2214            "timeout_seconds": timeout_seconds,
2215        });
2216        self.poll_request_json(
2217            "/worker/query-tasks/poll",
2218            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2219            &body,
2220            timeout + Duration::from_secs(5),
2221            transport_retries,
2222        )
2223        .await
2224    }
2225
2226    /// Complete a query task without appending workflow history.
2227    pub async fn complete_query_task<T: Serialize>(
2228        &self,
2229        query_task_id: &str,
2230        lease_owner: &str,
2231        query_task_attempt: u64,
2232        result: T,
2233        codec: &str,
2234    ) -> Result<Value> {
2235        let typed_result = AvroValue::from_serialize(&result)?;
2236        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
2237        self.complete_query_task_with_envelope(
2238            query_task_id,
2239            lease_owner,
2240            query_task_attempt,
2241            typed_result.into_json()?,
2242            result_envelope,
2243        )
2244        .await
2245    }
2246
2247    async fn complete_query_task_with_envelope(
2248        &self,
2249        query_task_id: &str,
2250        lease_owner: &str,
2251        query_task_attempt: u64,
2252        result: Value,
2253        result_envelope: Value,
2254    ) -> Result<Value> {
2255        let body = json!({
2256            "lease_owner": lease_owner,
2257            "query_task_attempt": query_task_attempt,
2258            "result": result,
2259            "result_envelope": result_envelope,
2260        });
2261        let path = format!("/worker/query-tasks/{query_task_id}/complete");
2262        let response = self
2263            .request_json(
2264                reqwest::Method::POST,
2265                &path,
2266                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2267                Some(&body),
2268            )
2269            .await;
2270        query_task_response(response)
2271    }
2272
2273    /// Report a stable machine-readable query-task failure.
2274    pub async fn fail_query_task(
2275        &self,
2276        query_task_id: &str,
2277        lease_owner: &str,
2278        query_task_attempt: u64,
2279        message: impl Into<String>,
2280        reason: impl Into<String>,
2281        failure_type: impl Into<String>,
2282    ) -> Result<Value> {
2283        let body = json!({
2284            "lease_owner": lease_owner,
2285            "query_task_attempt": query_task_attempt,
2286            "failure": {
2287                "message": message.into(),
2288                "reason": reason.into(),
2289                "type": failure_type.into(),
2290            }
2291        });
2292        let path = format!("/worker/query-tasks/{query_task_id}/fail");
2293        let response = self
2294            .request_json(
2295                reqwest::Method::POST,
2296                &path,
2297                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
2298                Some(&body),
2299            )
2300            .await;
2301        query_task_response(response)
2302    }
2303
2304    pub async fn heartbeat_worker(
2305        &self,
2306        worker_id: &str,
2307        workflow_available: usize,
2308        activity_available: usize,
2309    ) -> Result<Value> {
2310        let body = json!({
2311            "worker_id": worker_id,
2312            "task_slots": {
2313                "workflow_available": workflow_available,
2314                "activity_available": activity_available
2315            },
2316            "process_metrics": {
2317                "process_id": std::process::id(),
2318                "process_uptime_seconds": 0
2319            }
2320        });
2321
2322        self.request_json(
2323            reqwest::Method::POST,
2324            "/worker/heartbeat",
2325            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2326            Some(&body),
2327        )
2328        .await
2329    }
2330
2331    pub async fn poll_workflow_task(
2332        &self,
2333        worker_id: &str,
2334        task_queue: &str,
2335        timeout: Duration,
2336    ) -> Result<Option<WorkflowTask>> {
2337        Ok(self
2338            .poll_workflow_task_response(worker_id, task_queue, timeout)
2339            .await?
2340            .task)
2341    }
2342
2343    pub async fn poll_workflow_task_response(
2344        &self,
2345        worker_id: &str,
2346        task_queue: &str,
2347        timeout: Duration,
2348    ) -> Result<PollWorkflowTaskResponse> {
2349        let poll_request_id = unique_request_id("rust-workflow-poll");
2350        self.poll_workflow_task_response_with_request_id(
2351            worker_id,
2352            task_queue,
2353            timeout,
2354            &poll_request_id,
2355            1,
2356        )
2357        .await
2358    }
2359
2360    async fn poll_workflow_task_response_with_request_id(
2361        &self,
2362        worker_id: &str,
2363        task_queue: &str,
2364        timeout: Duration,
2365        poll_request_id: &str,
2366        transport_retries: usize,
2367    ) -> Result<PollWorkflowTaskResponse> {
2368        let body = json!({
2369            "worker_id": worker_id,
2370            "task_queue": task_queue,
2371            "poll_request_id": poll_request_id,
2372            "timeout_seconds": long_poll_timeout_seconds(timeout),
2373        });
2374        let mut data: PollWorkflowTaskResponse = self
2375            .poll_request_json(
2376                "/worker/workflow-tasks/poll",
2377                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2378                &body,
2379                timeout + Duration::from_secs(5),
2380                transport_retries,
2381            )
2382            .await?;
2383
2384        if let Some(task) = data.task.as_mut() {
2385            self.fetch_remaining_workflow_history(worker_id, task)
2386                .await?;
2387        }
2388
2389        Ok(data)
2390    }
2391
2392    async fn fetch_remaining_workflow_history(
2393        &self,
2394        worker_id: &str,
2395        task: &mut WorkflowTask,
2396    ) -> Result<()> {
2397        let mut next_token = task.next_history_page_token.clone();
2398
2399        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
2400            let lease_owner = task
2401                .lease_owner
2402                .clone()
2403                .unwrap_or_else(|| worker_id.to_string());
2404            let page = self
2405                .workflow_task_history_page(
2406                    &task.task_id,
2407                    &lease_owner,
2408                    task.workflow_task_attempt,
2409                    &token,
2410                )
2411                .await?;
2412
2413            task.append_history_page(page);
2414
2415            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
2416                return Err(Error::Codec(
2417                    "workflow history pagination returned the same page token".to_string(),
2418                ));
2419            }
2420
2421            next_token = task.next_history_page_token.clone();
2422        }
2423
2424        Ok(())
2425    }
2426
2427    async fn workflow_task_history_page(
2428        &self,
2429        task_id: &str,
2430        lease_owner: &str,
2431        workflow_task_attempt: u64,
2432        next_history_page_token: &str,
2433    ) -> Result<WorkflowTaskHistoryPage> {
2434        let body = json!({
2435            "lease_owner": lease_owner,
2436            "workflow_task_attempt": workflow_task_attempt,
2437            "next_history_page_token": next_history_page_token
2438        });
2439        let path = format!("/worker/workflow-tasks/{task_id}/history");
2440
2441        self.request_json(
2442            reqwest::Method::POST,
2443            &path,
2444            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2445            Some(&body),
2446        )
2447        .await
2448    }
2449
2450    pub async fn complete_workflow_task(
2451        &self,
2452        task_id: &str,
2453        lease_owner: &str,
2454        workflow_task_attempt: u64,
2455        commands: Vec<Value>,
2456    ) -> Result<Value> {
2457        let body = json!({
2458            "lease_owner": lease_owner,
2459            "workflow_task_attempt": workflow_task_attempt,
2460            "commands": commands
2461        });
2462        let path = format!("/worker/workflow-tasks/{task_id}/complete");
2463        self.request_json(
2464            reqwest::Method::POST,
2465            &path,
2466            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2467            Some(&body),
2468        )
2469        .await
2470    }
2471
2472    pub async fn fail_workflow_task(
2473        &self,
2474        task_id: &str,
2475        lease_owner: &str,
2476        workflow_task_attempt: u64,
2477        message: impl Into<String>,
2478    ) -> Result<Value> {
2479        self.fail_workflow_task_with_type(
2480            task_id,
2481            lease_owner,
2482            workflow_task_attempt,
2483            message,
2484            "RustWorkflowTaskFailure",
2485        )
2486        .await
2487    }
2488
2489    async fn fail_workflow_task_with_type(
2490        &self,
2491        task_id: &str,
2492        lease_owner: &str,
2493        workflow_task_attempt: u64,
2494        message: impl Into<String>,
2495        failure_type: &str,
2496    ) -> Result<Value> {
2497        let body = json!({
2498            "lease_owner": lease_owner,
2499            "workflow_task_attempt": workflow_task_attempt,
2500            "failure": {
2501                "message": message.into(),
2502                "type": failure_type
2503            }
2504        });
2505        let path = format!("/worker/workflow-tasks/{task_id}/fail");
2506        self.request_json(
2507            reqwest::Method::POST,
2508            &path,
2509            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2510            Some(&body),
2511        )
2512        .await
2513    }
2514
2515    pub async fn poll_activity_task(
2516        &self,
2517        worker_id: &str,
2518        task_queue: &str,
2519        timeout: Duration,
2520    ) -> Result<Option<ActivityTask>> {
2521        Ok(self
2522            .poll_activity_task_response(worker_id, task_queue, timeout)
2523            .await?
2524            .task)
2525    }
2526
2527    /// Poll an activity task while preserving server stop and drain metadata.
2528    pub async fn poll_activity_task_response(
2529        &self,
2530        worker_id: &str,
2531        task_queue: &str,
2532        timeout: Duration,
2533    ) -> Result<PollActivityTaskResponse> {
2534        let poll_request_id = unique_request_id("rust-activity-poll");
2535        self.poll_activity_task_response_with_request_id(
2536            worker_id,
2537            task_queue,
2538            timeout,
2539            &poll_request_id,
2540            1,
2541        )
2542        .await
2543    }
2544
2545    async fn poll_activity_task_response_with_request_id(
2546        &self,
2547        worker_id: &str,
2548        task_queue: &str,
2549        timeout: Duration,
2550        poll_request_id: &str,
2551        transport_retries: usize,
2552    ) -> Result<PollActivityTaskResponse> {
2553        let body = json!({
2554            "worker_id": worker_id,
2555            "task_queue": task_queue,
2556            "poll_request_id": poll_request_id,
2557            "timeout_seconds": long_poll_timeout_seconds(timeout),
2558        });
2559        let data: PollActivityTaskResponse = self
2560            .poll_request_json(
2561                "/worker/activity-tasks/poll",
2562                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2563                &body,
2564                timeout + Duration::from_secs(5),
2565                transport_retries,
2566            )
2567            .await?;
2568        Ok(data)
2569    }
2570
2571    pub async fn complete_activity_task<T: Serialize>(
2572        &self,
2573        task_id: &str,
2574        activity_attempt_id: &str,
2575        lease_owner: &str,
2576        result: T,
2577        codec: &str,
2578    ) -> Result<Value> {
2579        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
2580        let body = json!({
2581            "activity_attempt_id": activity_attempt_id,
2582            "lease_owner": lease_owner,
2583            "result": result
2584        });
2585        let path = format!("/worker/activity-tasks/{task_id}/complete");
2586        activity_task_response(
2587            self.request_json(
2588                reqwest::Method::POST,
2589                &path,
2590                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2591                Some(&body),
2592            )
2593            .await,
2594            "complete",
2595            task_id,
2596            activity_attempt_id,
2597        )
2598    }
2599
2600    pub async fn fail_activity_task(
2601        &self,
2602        task_id: &str,
2603        activity_attempt_id: &str,
2604        lease_owner: &str,
2605        message: impl Into<String>,
2606        non_retryable: bool,
2607    ) -> Result<Value> {
2608        let body = json!({
2609            "activity_attempt_id": activity_attempt_id,
2610            "lease_owner": lease_owner,
2611            "failure": {
2612                "message": message.into(),
2613                "type": "RustActivityFailure",
2614                "non_retryable": non_retryable
2615            }
2616        });
2617        let path = format!("/worker/activity-tasks/{task_id}/fail");
2618        activity_task_response(
2619            self.request_json(
2620                reqwest::Method::POST,
2621                &path,
2622                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2623                Some(&body),
2624            )
2625            .await,
2626            "fail",
2627            task_id,
2628            activity_attempt_id,
2629        )
2630    }
2631
2632    pub async fn heartbeat_activity_task<T: Serialize>(
2633        &self,
2634        task_id: &str,
2635        activity_attempt_id: &str,
2636        lease_owner: &str,
2637        details: T,
2638    ) -> Result<ActivityHeartbeatResponse> {
2639        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
2640        let body = json!({
2641            "activity_attempt_id": activity_attempt_id,
2642            "lease_owner": lease_owner,
2643            "details": details
2644        });
2645        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
2646        activity_task_response(
2647            self.request_json(
2648                reqwest::Method::POST,
2649                &path,
2650                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
2651                Some(&body),
2652            )
2653            .await,
2654            "heartbeat",
2655            task_id,
2656            activity_attempt_id,
2657        )
2658    }
2659
2660    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2661        &self,
2662        method: reqwest::Method,
2663        path: &str,
2664        protocol: RequestProtocol,
2665        body: Option<&B>,
2666    ) -> Result<T> {
2667        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
2668            .await
2669    }
2670
2671    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
2672        &self,
2673        method: reqwest::Method,
2674        path: &str,
2675        protocol: RequestProtocol,
2676        body: Option<&B>,
2677        timeout: Duration,
2678    ) -> Result<T> {
2679        let auth_token = self.auth_token(protocol)?;
2680        let mut request = self
2681            .http
2682            .request(method, format!("{}/api{}", self.base_url, path))
2683            .timeout(timeout)
2684            .header(reqwest::header::ACCEPT, "application/json")
2685            .header(reqwest::header::CONTENT_TYPE, "application/json")
2686            .header("X-Namespace", &self.namespace);
2687
2688        match protocol {
2689            RequestProtocol::Worker(version) => {
2690                request = request.header("X-Durable-Workflow-Protocol-Version", version);
2691            }
2692            RequestProtocol::ControlPlane => {
2693                request = request.header(
2694                    "X-Durable-Workflow-Control-Plane-Version",
2695                    CONTROL_PLANE_VERSION,
2696                );
2697            }
2698        }
2699
2700        if let Some(token) = auth_token {
2701            request = request.bearer_auth(token);
2702        }
2703
2704        if let Some(body) = body {
2705            request = request.json(body);
2706        }
2707
2708        let response = request.send().await?;
2709        let status = response.status();
2710        let bytes = response.bytes().await?;
2711
2712        if !status.is_success() {
2713            let body = String::from_utf8_lossy(&bytes).to_string();
2714            if let Some(protocol) = protocol_failure(status, &body) {
2715                return Err(Error::Protocol(protocol));
2716            }
2717            return Err(Error::Http { status, body });
2718        }
2719
2720        if bytes.is_empty() {
2721            return Ok(serde_json::from_value(Value::Null)?);
2722        }
2723
2724        Ok(serde_json::from_slice(&bytes)?)
2725    }
2726
2727    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
2728        &self,
2729        path: &str,
2730        protocol: RequestProtocol,
2731        body: &B,
2732        timeout: Duration,
2733        max_retries: usize,
2734    ) -> Result<T> {
2735        let mut retries = 0;
2736
2737        loop {
2738            let response = self
2739                .request_json_with_timeout(
2740                    reqwest::Method::POST,
2741                    path,
2742                    protocol,
2743                    Some(body),
2744                    timeout,
2745                )
2746                .await;
2747
2748            match response {
2749                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
2750                response => return worker_poll_response(response),
2751            }
2752        }
2753    }
2754
2755    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
2756        match protocol {
2757            RequestProtocol::Worker(_) => {
2758                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
2759                    return Ok(Some(token));
2760                }
2761                if self.control_token.is_some() {
2762                    return Err(Error::MissingRoleCredentials {
2763                        role: "worker",
2764                        opposite_role: "control",
2765                    });
2766                }
2767                Ok(None)
2768            }
2769            RequestProtocol::ControlPlane => {
2770                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
2771                    return Ok(Some(token));
2772                }
2773                if self.worker_token.is_some() {
2774                    return Err(Error::MissingRoleCredentials {
2775                        role: "control",
2776                        opposite_role: "worker",
2777                    });
2778                }
2779                Ok(None)
2780            }
2781        }
2782    }
2783}
2784
2785fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
2786    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2787    let reason = body
2788        .get("reason")
2789        .and_then(Value::as_str)
2790        .unwrap_or("query_rejected")
2791        .to_string();
2792    let message = body
2793        .get("message")
2794        .or_else(|| body.get("error"))
2795        .and_then(Value::as_str)
2796        .unwrap_or("workflow query was rejected")
2797        .to_string();
2798
2799    QueryFailure {
2800        status: status.as_u16(),
2801        reason,
2802        message,
2803        body,
2804    }
2805}
2806
2807fn workflow_command_result(
2808    command: WorkflowCommandKind,
2809    data: Value,
2810    workflow_id: &str,
2811    run_id: Option<&str>,
2812) -> WorkflowCommandResult {
2813    WorkflowCommandResult {
2814        command,
2815        workflow_id: data
2816            .get("workflow_id")
2817            .and_then(Value::as_str)
2818            .unwrap_or(workflow_id)
2819            .to_string(),
2820        run_id: data
2821            .get("run_id")
2822            .and_then(Value::as_str)
2823            .or(run_id)
2824            .map(str::to_string),
2825        outcome: data
2826            .get("outcome")
2827            .and_then(Value::as_str)
2828            .map(str::to_string),
2829        reason: data
2830            .get("reason")
2831            .and_then(Value::as_str)
2832            .map(str::to_string),
2833        command_status: data
2834            .get("command_status")
2835            .and_then(Value::as_str)
2836            .map(str::to_string),
2837        raw: data,
2838    }
2839}
2840
2841fn workflow_command_rejection(
2842    command: WorkflowCommandKind,
2843    status: reqwest::StatusCode,
2844    raw_body: String,
2845    workflow_id: &str,
2846    run_id: Option<&str>,
2847) -> WorkflowCommandRejection {
2848    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
2849    WorkflowCommandRejection {
2850        command,
2851        status: status.as_u16(),
2852        reason: body
2853            .get("reason")
2854            .and_then(Value::as_str)
2855            .unwrap_or("workflow_command_rejected")
2856            .to_string(),
2857        message: body
2858            .get("message")
2859            .or_else(|| body.get("error"))
2860            .and_then(Value::as_str)
2861            .unwrap_or("workflow lifecycle command was rejected")
2862            .to_string(),
2863        workflow_id: body
2864            .get("workflow_id")
2865            .and_then(Value::as_str)
2866            .unwrap_or(workflow_id)
2867            .to_string(),
2868        run_id: body
2869            .get("run_id")
2870            .and_then(Value::as_str)
2871            .or(run_id)
2872            .map(str::to_string),
2873        target_scope: body
2874            .get("target_scope")
2875            .and_then(Value::as_str)
2876            .map(str::to_string),
2877        body,
2878    }
2879}
2880
2881fn query_task_response(response: Result<Value>) -> Result<Value> {
2882    match response {
2883        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
2884        response => response,
2885    }
2886}
2887
2888fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
2889    match response {
2890        Err(Error::Http { status, body })
2891            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
2892        {
2893            Ok(serde_json::from_str(&body)?)
2894        }
2895        response => response,
2896    }
2897}
2898
2899fn worker_poll_body_is_stop(body: &str) -> bool {
2900    serde_json::from_str::<Value>(body)
2901        .ok()
2902        .is_some_and(|body| {
2903            worker_poll_is_stop(
2904                body.get("poll_status").and_then(Value::as_str),
2905                body.get("reason").and_then(Value::as_str),
2906            )
2907        })
2908}
2909
2910fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
2911    matches!(poll_status, Some("draining" | "stopped"))
2912        || matches!(reason, Some("worker_draining" | "worker_stopped"))
2913}
2914
2915fn query_task_rejection_is_final(error: &Error) -> bool {
2916    matches!(
2917        error,
2918        Error::QueryFailed(failure)
2919            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
2920    )
2921}
2922
2923fn activity_task_response<T>(
2924    response: Result<T>,
2925    operation: &str,
2926    task_id: &str,
2927    activity_attempt_id: &str,
2928) -> Result<T> {
2929    match response {
2930        Err(Error::Http { status, body }) => {
2931            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
2932            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
2933                operation: operation.to_string(),
2934                status: status.as_u16(),
2935                reason: body
2936                    .get("reason")
2937                    .and_then(Value::as_str)
2938                    .unwrap_or("activity_task_rejected")
2939                    .to_string(),
2940                task_id: body
2941                    .get("task_id")
2942                    .and_then(Value::as_str)
2943                    .unwrap_or(task_id)
2944                    .to_string(),
2945                activity_attempt_id: body
2946                    .get("activity_attempt_id")
2947                    .and_then(Value::as_str)
2948                    .unwrap_or(activity_attempt_id)
2949                    .to_string(),
2950                cancel_requested: body
2951                    .get("cancel_requested")
2952                    .and_then(Value::as_bool)
2953                    .unwrap_or(false),
2954                can_continue: body.get("can_continue").and_then(Value::as_bool),
2955                run_closed_reason: body
2956                    .get("run_closed_reason")
2957                    .and_then(Value::as_str)
2958                    .map(str::to_string),
2959                body,
2960            }))
2961        }
2962        response => response,
2963    }
2964}
2965
2966fn activity_task_rejection_is_final(error: &Error) -> bool {
2967    matches!(
2968        error,
2969        Error::ActivityTaskRejected(rejection)
2970            if matches!(
2971                rejection.reason.as_str(),
2972                "run_cancelled"
2973                    | "run_terminated"
2974                    | "attempt_closed"
2975                    | "stale_attempt"
2976                    | "activity_cancelled"
2977                    | "task_cancelled"
2978                    | "run_closed"
2979                    | "activity_not_running"
2980                    | "attempt_not_found"
2981            )
2982    )
2983}
2984
2985fn workflow_task_completion_is_terminal_timeout(
2986    error: &Error,
2987    task_id: &str,
2988    workflow_task_attempt: u64,
2989    run_id: Option<&str>,
2990) -> bool {
2991    let Error::Http { status, body } = error else {
2992        return false;
2993    };
2994    if *status != reqwest::StatusCode::CONFLICT {
2995        return false;
2996    }
2997
2998    let Some(run_id) = run_id else {
2999        return false;
3000    };
3001    let Ok(body) = serde_json::from_str::<Value>(body) else {
3002        return false;
3003    };
3004
3005    body.get("recorded").and_then(Value::as_bool) == Some(false)
3006        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
3007        && body.get("run_status").and_then(Value::as_str) == Some("failed")
3008        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
3009        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
3010        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
3011}
3012
3013fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
3014    let body: Value = serde_json::from_str(raw_body).ok()?;
3015    let reason = body.get("reason")?.as_str()?;
3016    if !matches!(
3017        reason,
3018        "missing_protocol_version"
3019            | "unsupported_protocol_version"
3020            | "missing_control_plane_version"
3021            | "unsupported_control_plane_version"
3022    ) {
3023        return None;
3024    }
3025
3026    Some(ProtocolFailure {
3027        status: status.as_u16(),
3028        reason: reason.to_string(),
3029        message: body
3030            .get("message")
3031            .or_else(|| body.get("error"))
3032            .and_then(Value::as_str)
3033            .unwrap_or("protocol version rejected")
3034            .to_string(),
3035        supported_version: body
3036            .get("supported_version")
3037            .and_then(Value::as_str)
3038            .map(str::to_string),
3039        requested_version: body
3040            .get("requested_version")
3041            .and_then(Value::as_str)
3042            .map(str::to_string),
3043        body,
3044    })
3045}
3046
3047fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
3048    timeout
3049        .as_secs()
3050        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
3051        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
3052}
3053
3054fn worker_operation_is_retryable(error: &Error) -> bool {
3055    match error {
3056        Error::Transport(error) => {
3057            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
3058        }
3059        Error::Http { status, .. } => {
3060            matches!(
3061                *status,
3062                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
3063            ) || status.is_server_error()
3064        }
3065        _ => false,
3066    }
3067}
3068
3069fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
3070    let exponent = retry.saturating_sub(1).min(31) as u32;
3071    policy
3072        .initial_backoff
3073        .saturating_mul(1_u32 << exponent)
3074        .min(policy.max_backoff)
3075}
3076
3077#[derive(Debug)]
3078pub struct ClientBuilder {
3079    base_url: String,
3080    token: Option<String>,
3081    control_token: Option<String>,
3082    worker_token: Option<String>,
3083    namespace: String,
3084    timeout: Duration,
3085}
3086
3087impl ClientBuilder {
3088    pub fn token(mut self, token: Option<String>) -> Self {
3089        self.token = token;
3090        self
3091    }
3092
3093    pub fn control_token(mut self, token: Option<String>) -> Self {
3094        self.control_token = token;
3095        self
3096    }
3097
3098    pub fn worker_token(mut self, token: Option<String>) -> Self {
3099        self.worker_token = token;
3100        self
3101    }
3102
3103    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
3104        self.namespace = namespace.into();
3105        self
3106    }
3107
3108    pub fn timeout(mut self, timeout: Duration) -> Self {
3109        self.timeout = timeout;
3110        self
3111    }
3112
3113    pub fn build(self) -> Result<Client> {
3114        let base_url = self.base_url.trim_end_matches('/').to_string();
3115        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
3116            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
3117            .unwrap_or_else(|_| base_url.ends_with("/api"));
3118
3119        if has_sdk_api_suffix {
3120            return Err(Error::InvalidBaseUrl);
3121        }
3122
3123        Ok(Client {
3124            http: reqwest::Client::builder().timeout(self.timeout).build()?,
3125            base_url,
3126            token: self.token,
3127            control_token: self.control_token,
3128            worker_token: self.worker_token,
3129            namespace: self.namespace,
3130        })
3131    }
3132}
3133
3134#[derive(Clone, Debug)]
3135pub struct WorkflowHandle {
3136    client: Client,
3137    pub workflow_id: String,
3138    pub run_id: Option<String>,
3139    pub workflow_type: String,
3140}
3141
3142impl WorkflowHandle {
3143    /// Describe whichever run is current for this stable workflow instance.
3144    pub async fn describe(&self) -> Result<WorkflowDescription> {
3145        self.client.describe_workflow(&self.workflow_id).await
3146    }
3147
3148    /// Describe the run identity originally selected by this handle.
3149    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
3150        let run_id = self.run_id.as_deref().ok_or_else(|| {
3151            Error::Codec("run_id is required for selected-run description".to_string())
3152        })?;
3153        self.client
3154            .describe_workflow_run(&self.workflow_id, run_id)
3155            .await
3156    }
3157
3158    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
3159        self.client
3160            .signal_workflow(&self.workflow_id, signal_name, input)
3161            .await
3162    }
3163
3164    /// Signal only if this handle's selected run is still current.
3165    pub async fn signal_selected_run<T: Serialize>(
3166        &self,
3167        signal_name: &str,
3168        input: T,
3169    ) -> Result<Value> {
3170        let run_id = self.run_id.as_deref().ok_or_else(|| {
3171            Error::Codec("run_id is required for selected-run signaling".to_string())
3172        })?;
3173        self.client
3174            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
3175            .await
3176    }
3177
3178    /// Request cooperative cancellation of whichever run is current.
3179    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
3180        self.client
3181            .cancel_workflow(&self.workflow_id, options)
3182            .await
3183    }
3184
3185    /// Request cancellation only if this handle's selected run is still current.
3186    pub async fn cancel_selected_run(
3187        &self,
3188        options: WorkflowCommandOptions,
3189    ) -> Result<WorkflowCommandResult> {
3190        let run_id = self.run_id.as_deref().ok_or_else(|| {
3191            Error::Codec("run_id is required for selected-run cancellation".to_string())
3192        })?;
3193        self.client
3194            .cancel_workflow_run(&self.workflow_id, run_id, options)
3195            .await
3196    }
3197
3198    /// Forcefully terminate whichever run is current.
3199    pub async fn terminate(
3200        &self,
3201        options: WorkflowCommandOptions,
3202    ) -> Result<WorkflowCommandResult> {
3203        self.client
3204            .terminate_workflow(&self.workflow_id, options)
3205            .await
3206    }
3207
3208    /// Terminate only if this handle's selected run is still current.
3209    pub async fn terminate_selected_run(
3210        &self,
3211        options: WorkflowCommandOptions,
3212    ) -> Result<WorkflowCommandResult> {
3213        let run_id = self.run_id.as_deref().ok_or_else(|| {
3214            Error::Codec("run_id is required for selected-run termination".to_string())
3215        })?;
3216        self.client
3217            .terminate_workflow_run(&self.workflow_id, run_id, options)
3218            .await
3219    }
3220
3221    /// Execute a named, read-only query against this workflow.
3222    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
3223        self.client
3224            .query_workflow(&self.workflow_id, query_name, input)
3225            .await
3226    }
3227
3228    pub async fn query_avro_value<T: Serialize>(
3229        &self,
3230        query_name: &str,
3231        input: T,
3232    ) -> Result<AvroValue> {
3233        self.client
3234            .query_workflow_avro_value(&self.workflow_id, query_name, input)
3235            .await
3236    }
3237
3238    pub async fn update<T: Serialize>(
3239        &self,
3240        update_name: &str,
3241        input: T,
3242        request_id: Option<&str>,
3243    ) -> Result<Value> {
3244        self.client
3245            .update_workflow(&self.workflow_id, update_name, input, request_id)
3246            .await
3247    }
3248
3249    pub async fn update_avro_value<T: Serialize>(
3250        &self,
3251        update_name: &str,
3252        input: T,
3253        request_id: Option<&str>,
3254    ) -> Result<AvroValue> {
3255        self.client
3256            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
3257            .await
3258    }
3259
3260    /// Query only if this handle's selected run is still current.
3261    pub async fn query_selected_run<T: Serialize>(
3262        &self,
3263        query_name: &str,
3264        input: T,
3265    ) -> Result<Value> {
3266        let run_id = self
3267            .run_id
3268            .as_deref()
3269            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
3270        self.client
3271            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
3272            .await
3273    }
3274
3275    /// Await the final terminal outcome of the current continue-as-new chain.
3276    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
3277        self.result_target(options, None).await
3278    }
3279
3280    /// Await the final result without projecting Avro bytes through JSON.
3281    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
3282        self.result_avro_value_target(options, None).await
3283    }
3284
3285    /// Await only the run identity originally selected by this handle.
3286    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
3287        let run_id = self.run_id.as_deref().ok_or_else(|| {
3288            Error::Codec("run_id is required for selected-run result".to_string())
3289        })?;
3290        self.result_target(options, Some(run_id)).await
3291    }
3292
3293    /// Await the selected run's result on the lossless Avro Value surface.
3294    pub async fn result_selected_run_avro_value(
3295        &self,
3296        options: WorkflowResultOptions,
3297    ) -> Result<AvroValue> {
3298        let run_id = self.run_id.as_deref().ok_or_else(|| {
3299            Error::Codec("run_id is required for selected-run result".to_string())
3300        })?;
3301        self.result_avro_value_target(options, Some(run_id)).await
3302    }
3303
3304    async fn result_avro_value_target(
3305        &self,
3306        options: WorkflowResultOptions,
3307        selected_run_id: Option<&str>,
3308    ) -> Result<AvroValue> {
3309        let started = Instant::now();
3310
3311        loop {
3312            let description = match selected_run_id {
3313                Some(run_id) => {
3314                    self.client
3315                        .describe_workflow_run(&self.workflow_id, run_id)
3316                        .await?
3317                }
3318                None => self.describe().await?,
3319            };
3320            if description.is_completed() {
3321                return description.output_avro_value.ok_or_else(|| {
3322                    Error::Codec(
3323                        "missing_payload_envelope: typed workflow result requires output_envelope"
3324                            .to_string(),
3325                    )
3326                });
3327            }
3328            if description.is_terminal() {
3329                let outcome =
3330                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3331                return Err(match outcome.kind {
3332                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3333                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3334                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3335                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3336                });
3337            }
3338            if started.elapsed() >= options.timeout {
3339                return Err(Error::Timeout);
3340            }
3341            tokio::time::sleep(options.poll_interval).await;
3342        }
3343    }
3344
3345    async fn result_target(
3346        &self,
3347        options: WorkflowResultOptions,
3348        selected_run_id: Option<&str>,
3349    ) -> Result<Value> {
3350        let started = Instant::now();
3351
3352        loop {
3353            let description = match selected_run_id {
3354                Some(run_id) => {
3355                    self.client
3356                        .describe_workflow_run(&self.workflow_id, run_id)
3357                        .await?
3358                }
3359                None => self.describe().await?,
3360            };
3361            if description.is_completed() {
3362                return Ok(description.output.unwrap_or(Value::Null));
3363            }
3364
3365            if description.is_terminal() {
3366                let outcome =
3367                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
3368                return Err(match outcome.kind {
3369                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
3370                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
3371                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
3372                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
3373                });
3374            }
3375
3376            if started.elapsed() >= options.timeout {
3377                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
3378                    kind: WorkflowTerminalKind::TimedOut,
3379                    workflow_id: description
3380                        .workflow_id
3381                        .clone()
3382                        .unwrap_or_else(|| self.workflow_id.clone()),
3383                    run_id: description
3384                        .run_id
3385                        .clone()
3386                        .or_else(|| selected_run_id.map(str::to_string)),
3387                    reason: "result_wait_timeout".to_string(),
3388                    failure_category: Some("client_timeout".to_string()),
3389                    failure_id: None,
3390                    exception_type: None,
3391                    exception_class: None,
3392                    non_retryable: None,
3393                    message: Some(format!(
3394                        "workflow result was not terminal within {:?}",
3395                        options.timeout
3396                    )),
3397                    exception: None,
3398                    raw: description.raw_value(),
3399                }));
3400            }
3401
3402            tokio::time::sleep(options.poll_interval).await;
3403        }
3404    }
3405}
3406
3407#[derive(Clone, Copy, Debug)]
3408pub struct WorkflowResultOptions {
3409    pub poll_interval: Duration,
3410    pub timeout: Duration,
3411}
3412
3413impl Default for WorkflowResultOptions {
3414    fn default() -> Self {
3415        Self {
3416            poll_interval: Duration::from_millis(500),
3417            timeout: Duration::from_secs(30),
3418        }
3419    }
3420}
3421
3422#[derive(Clone, Debug, Deserialize)]
3423pub struct WorkflowDescription {
3424    pub workflow_id: Option<String>,
3425    pub run_id: Option<String>,
3426    pub workflow_type: Option<String>,
3427    pub status: Option<String>,
3428    #[serde(default)]
3429    pub closed_reason: Option<String>,
3430    #[serde(default)]
3431    pub error: Option<String>,
3432    #[serde(default)]
3433    pub failure: Option<Value>,
3434    #[serde(default)]
3435    pub exception: Option<Value>,
3436    #[serde(default)]
3437    pub failures: Vec<Value>,
3438    #[serde(default)]
3439    pub output: Option<Value>,
3440    #[serde(default)]
3441    pub output_envelope: Option<Value>,
3442    #[serde(skip)]
3443    pub output_avro_value: Option<AvroValue>,
3444    #[serde(flatten)]
3445    pub raw: HashMap<String, Value>,
3446}
3447
3448impl WorkflowDescription {
3449    pub fn is_completed(&self) -> bool {
3450        matches!(self.status.as_deref(), Some("completed" | "Completed"))
3451    }
3452
3453    pub fn is_terminal(&self) -> bool {
3454        matches!(
3455            self.status.as_deref(),
3456            Some(
3457                "completed"
3458                    | "Completed"
3459                    | "failed"
3460                    | "Failed"
3461                    | "cancelled"
3462                    | "Cancelled"
3463                    | "terminated"
3464                    | "Terminated"
3465                    | "timed_out"
3466                    | "TimedOut",
3467            )
3468        )
3469    }
3470
3471    fn decode_payloads(&mut self) -> Result<()> {
3472        if let Some(envelope) = &self.output_envelope {
3473            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
3474            self.output = Some(value.clone().into_json()?);
3475            self.output_avro_value = Some(value);
3476        }
3477
3478        Ok(())
3479    }
3480
3481    fn raw_value(&self) -> Value {
3482        let mut data = self.raw.clone();
3483        data.insert(
3484            "workflow_id".to_string(),
3485            self.workflow_id
3486                .clone()
3487                .map(Value::String)
3488                .unwrap_or(Value::Null),
3489        );
3490        data.insert(
3491            "run_id".to_string(),
3492            self.run_id
3493                .clone()
3494                .map(Value::String)
3495                .unwrap_or(Value::Null),
3496        );
3497        data.insert(
3498            "workflow_type".to_string(),
3499            self.workflow_type
3500                .clone()
3501                .map(Value::String)
3502                .unwrap_or(Value::Null),
3503        );
3504        data.insert(
3505            "status".to_string(),
3506            self.status
3507                .clone()
3508                .map(Value::String)
3509                .unwrap_or(Value::Null),
3510        );
3511        data.insert(
3512            "closed_reason".to_string(),
3513            self.closed_reason
3514                .clone()
3515                .map(Value::String)
3516                .unwrap_or(Value::Null),
3517        );
3518        if let Some(failure) = &self.failure {
3519            data.insert("failure".to_string(), failure.clone());
3520        }
3521        if let Some(exception) = &self.exception {
3522            data.insert("exception".to_string(), exception.clone());
3523        }
3524        Value::Object(data.into_iter().collect())
3525    }
3526}
3527
3528fn workflow_terminal_outcome(
3529    description: &WorkflowDescription,
3530    workflow_id: &str,
3531    run_id: Option<&str>,
3532) -> WorkflowTerminalOutcome {
3533    let terminal_kind = description
3534        .closed_reason
3535        .as_deref()
3536        .or(description.status.as_deref())
3537        .unwrap_or("failed")
3538        .to_ascii_lowercase();
3539    let kind = match terminal_kind.as_str() {
3540        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
3541        "terminated" => WorkflowTerminalKind::Terminated,
3542        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
3543        _ => WorkflowTerminalKind::Failed,
3544    };
3545    let default_reason = match kind {
3546        WorkflowTerminalKind::Failed => "workflow_failed",
3547        WorkflowTerminalKind::Cancelled => "cancelled",
3548        WorkflowTerminalKind::Terminated => "terminated",
3549        WorkflowTerminalKind::TimedOut => "timed_out",
3550    };
3551    let failure = description
3552        .failure
3553        .as_ref()
3554        .filter(|value| value.is_object());
3555    let nested_failure = failure
3556        .and_then(|value| value.get("failures"))
3557        .and_then(Value::as_array)
3558        .and_then(|failures| failures.last())
3559        .or_else(|| description.failures.last());
3560    let exception = description
3561        .exception
3562        .clone()
3563        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
3564        .or_else(|| {
3565            nested_failure
3566                .and_then(|value| value.get("exception_payload"))
3567                .cloned()
3568        });
3569    let string_field = |name: &str| {
3570        failure
3571            .and_then(|value| value.get(name))
3572            .and_then(Value::as_str)
3573            .or_else(|| {
3574                nested_failure
3575                    .and_then(|value| value.get(name))
3576                    .and_then(Value::as_str)
3577            })
3578            .map(str::to_string)
3579    };
3580    let exception_field = |name: &str| {
3581        exception
3582            .as_ref()
3583            .and_then(|value| value.get(name))
3584            .and_then(Value::as_str)
3585            .map(str::to_string)
3586    };
3587    let message = description
3588        .error
3589        .clone()
3590        .or_else(|| string_field("message"))
3591        .or_else(|| exception_field("message"));
3592    let reason = description
3593        .raw
3594        .get("reason")
3595        .and_then(Value::as_str)
3596        .map(str::to_string)
3597        .or_else(|| {
3598            failure
3599                .and_then(|value| value.get("reason"))
3600                .and_then(Value::as_str)
3601                .map(str::to_string)
3602        })
3603        .or_else(|| description.closed_reason.clone())
3604        .unwrap_or_else(|| default_reason.to_string());
3605    let failure_id = string_field("failure_id").or_else(|| {
3606        nested_failure
3607            .and_then(|value| value.get("id"))
3608            .and_then(Value::as_str)
3609            .map(str::to_string)
3610    });
3611
3612    WorkflowTerminalOutcome {
3613        kind,
3614        workflow_id: description
3615            .workflow_id
3616            .clone()
3617            .unwrap_or_else(|| workflow_id.to_string()),
3618        run_id: description
3619            .run_id
3620            .clone()
3621            .or_else(|| run_id.map(str::to_string)),
3622        reason,
3623        failure_category: string_field("failure_category")
3624            .or_else(|| Some(default_reason.to_string())),
3625        failure_id,
3626        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
3627        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
3628        non_retryable: failure
3629            .and_then(|value| value.get("non_retryable"))
3630            .and_then(Value::as_bool)
3631            .or_else(|| {
3632                nested_failure
3633                    .and_then(|value| value.get("non_retryable"))
3634                    .and_then(Value::as_bool)
3635            }),
3636        message,
3637        exception,
3638        raw: description.raw_value(),
3639    }
3640}
3641
3642#[derive(Clone, Debug, Deserialize)]
3643pub struct RegisterWorkerResponse {
3644    pub worker_id: String,
3645    pub registered: bool,
3646    #[serde(default)]
3647    pub heartbeat_interval_seconds: Option<u64>,
3648    #[serde(default)]
3649    pub protocol_version: Option<String>,
3650    #[serde(default)]
3651    pub server_capabilities: Option<Value>,
3652}
3653
3654/// Result of gracefully removing a worker-plane registration.
3655#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
3656pub struct WorkerDeregistrationEnvelope {
3657    pub worker_id: String,
3658    pub outcome: String,
3659    pub recovered_workflow_task_count: u64,
3660}
3661
3662#[derive(Clone, Debug, Deserialize)]
3663pub struct PollWorkflowTaskResponse {
3664    #[serde(default)]
3665    pub task: Option<WorkflowTask>,
3666    #[serde(default)]
3667    pub poll_status: Option<String>,
3668    #[serde(default)]
3669    pub reason: Option<String>,
3670    #[serde(default)]
3671    pub protocol_version: Option<String>,
3672    #[serde(default)]
3673    pub server_capabilities: Option<Value>,
3674}
3675
3676impl PollWorkflowTaskResponse {
3677    /// Classify this response without parsing server display text.
3678    pub fn outcome(&self) -> WorkerPollOutcome {
3679        worker_poll_outcome(
3680            self.task.is_some(),
3681            self.poll_status.as_deref(),
3682            self.reason.as_deref(),
3683        )
3684    }
3685}
3686
3687#[derive(Clone, Debug, Deserialize)]
3688pub struct PollActivityTaskResponse {
3689    #[serde(default)]
3690    pub task: Option<ActivityTask>,
3691    #[serde(default)]
3692    pub poll_status: Option<String>,
3693    #[serde(default)]
3694    pub reason: Option<String>,
3695}
3696
3697impl PollActivityTaskResponse {
3698    /// Classify this response without parsing server display text.
3699    pub fn outcome(&self) -> WorkerPollOutcome {
3700        worker_poll_outcome(
3701            self.task.is_some(),
3702            self.poll_status.as_deref(),
3703            self.reason.as_deref(),
3704        )
3705    }
3706}
3707
3708#[derive(Clone, Debug, Deserialize)]
3709pub struct PollQueryTaskResponse {
3710    #[serde(default)]
3711    pub task: Option<QueryTask>,
3712    #[serde(default)]
3713    pub poll_status: Option<String>,
3714    #[serde(default)]
3715    pub reason: Option<String>,
3716}
3717
3718impl PollQueryTaskResponse {
3719    /// Classify this response without parsing server display text.
3720    pub fn outcome(&self) -> WorkerPollOutcome {
3721        worker_poll_outcome(
3722            self.task.is_some(),
3723            self.poll_status.as_deref(),
3724            self.reason.as_deref(),
3725        )
3726    }
3727}
3728
3729/// Stable classification for worker poll responses.
3730#[derive(Clone, Debug, PartialEq, Eq)]
3731pub enum WorkerPollOutcome {
3732    /// A task was leased and is available on the response.
3733    Task,
3734    /// No task was leased, but the worker should continue polling.
3735    Idle {
3736        poll_status: Option<String>,
3737        reason: Option<String>,
3738    },
3739    /// The server asked this worker to stop claiming new work.
3740    Stop {
3741        poll_status: Option<String>,
3742        reason: Option<String>,
3743    },
3744}
3745
3746impl WorkerPollOutcome {
3747    pub fn should_stop(&self) -> bool {
3748        matches!(self, Self::Stop { .. })
3749    }
3750}
3751
3752fn worker_poll_outcome(
3753    has_task: bool,
3754    poll_status: Option<&str>,
3755    reason: Option<&str>,
3756) -> WorkerPollOutcome {
3757    if worker_poll_is_stop(poll_status, reason) {
3758        return WorkerPollOutcome::Stop {
3759            poll_status: poll_status.map(str::to_string),
3760            reason: reason.map(str::to_string),
3761        };
3762    }
3763
3764    if has_task {
3765        WorkerPollOutcome::Task
3766    } else {
3767        WorkerPollOutcome::Idle {
3768            poll_status: poll_status.map(str::to_string),
3769            reason: reason.map(str::to_string),
3770        }
3771    }
3772}
3773
3774/// An ephemeral server-routed query task.
3775#[derive(Clone, Debug, Deserialize)]
3776pub struct QueryTask {
3777    pub query_task_id: String,
3778    #[serde(default = "default_workflow_task_attempt")]
3779    pub query_task_attempt: u64,
3780    #[serde(default)]
3781    pub lease_owner: Option<String>,
3782    #[serde(default)]
3783    pub workflow_id: Option<String>,
3784    #[serde(default)]
3785    pub run_id: Option<String>,
3786    pub workflow_type: String,
3787    pub query_name: String,
3788    #[serde(default = "default_payload_codec")]
3789    pub payload_codec: String,
3790    #[serde(default)]
3791    pub workflow_arguments: Option<Value>,
3792    #[serde(default)]
3793    pub query_arguments: Option<Value>,
3794    #[serde(default)]
3795    pub history_events: Vec<HistoryEvent>,
3796    #[serde(default)]
3797    pub history_export: Option<Value>,
3798    #[serde(default)]
3799    pub run_status: Option<String>,
3800}
3801
3802#[derive(Clone, Debug, Deserialize)]
3803pub struct WorkflowTask {
3804    pub task_id: String,
3805    #[serde(default)]
3806    pub workflow_id: Option<String>,
3807    #[serde(default)]
3808    pub run_id: Option<String>,
3809    pub workflow_type: String,
3810    #[serde(default = "default_payload_codec")]
3811    pub payload_codec: String,
3812    #[serde(default)]
3813    pub arguments: Option<Value>,
3814    #[serde(default)]
3815    pub history_events: Vec<HistoryEvent>,
3816    #[serde(default)]
3817    pub total_history_events: Option<u64>,
3818    #[serde(default)]
3819    pub history_size_bytes: Option<u64>,
3820    #[serde(default)]
3821    pub continue_as_new_recommended: Option<bool>,
3822    #[serde(default)]
3823    pub history_budget_pressure: Option<String>,
3824    #[serde(default)]
3825    pub next_history_page_token: Option<String>,
3826    #[serde(default = "default_workflow_task_attempt")]
3827    pub workflow_task_attempt: u64,
3828    #[serde(default)]
3829    pub workflow_signal_id: Option<String>,
3830    #[serde(default)]
3831    pub signal_name: Option<String>,
3832    #[serde(default)]
3833    pub signal_arguments: Option<Value>,
3834    #[serde(default)]
3835    pub workflow_update_id: Option<String>,
3836    #[serde(default)]
3837    pub update_name: Option<String>,
3838    #[serde(default)]
3839    pub lease_owner: Option<String>,
3840}
3841
3842impl WorkflowTask {
3843    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
3844        self.history_events.extend(page.history_events);
3845
3846        if page.total_history_events.is_some() {
3847            self.total_history_events = page.total_history_events;
3848        }
3849
3850        self.next_history_page_token = page
3851            .next_history_page_token
3852            .filter(|token| !token.is_empty());
3853    }
3854}
3855
3856#[derive(Clone, Debug, Deserialize)]
3857struct WorkflowTaskHistoryPage {
3858    #[serde(default)]
3859    history_events: Vec<HistoryEvent>,
3860    #[serde(default)]
3861    total_history_events: Option<u64>,
3862    #[serde(default)]
3863    next_history_page_token: Option<String>,
3864}
3865
3866#[derive(Clone, Debug, Deserialize)]
3867pub struct ActivityTask {
3868    pub task_id: String,
3869    #[serde(default)]
3870    pub activity_attempt_id: Option<String>,
3871    #[serde(default)]
3872    pub attempt_id: Option<String>,
3873    pub activity_type: String,
3874    #[serde(default = "default_payload_codec")]
3875    pub payload_codec: String,
3876    #[serde(default)]
3877    pub arguments: Option<Value>,
3878    #[serde(default = "default_attempt_number")]
3879    pub attempt_number: u64,
3880    #[serde(default)]
3881    pub lease_owner: Option<String>,
3882}
3883
3884#[derive(Clone, Debug, Deserialize)]
3885pub struct HistoryEvent {
3886    #[serde(alias = "type")]
3887    pub event_type: String,
3888    #[serde(default)]
3889    pub payload: Value,
3890    #[serde(flatten)]
3891    pub raw: HashMap<String, Value>,
3892}
3893
3894/// One decoded signal in the committed workflow-history snapshot.
3895#[derive(Clone, Debug, PartialEq)]
3896pub struct QuerySignal {
3897    pub id: Option<String>,
3898    pub name: String,
3899    pub arguments: Vec<Value>,
3900    avro_arguments: Vec<AvroValue>,
3901    pub workflow_sequence: Option<u64>,
3902}
3903
3904impl QuerySignal {
3905    /// Lossless fixed Avro Value arguments for this committed signal.
3906    pub fn arguments_avro_value(&self) -> &[AvroValue] {
3907        &self.avro_arguments
3908    }
3909}
3910
3911/// Immutable state supplied to a registered query handler.
3912///
3913/// This context intentionally exposes no activity, signal-wait, or command
3914/// APIs. Query handlers inspect committed history and return a value; query
3915/// completion does not append an event or advance deterministic execution.
3916#[derive(Clone, Debug)]
3917pub struct QueryContext {
3918    pub workflow_id: Option<String>,
3919    pub run_id: Option<String>,
3920    pub workflow_type: String,
3921    pub run_status: Option<String>,
3922    workflow_input: Value,
3923    workflow_input_avro_value: AvroValue,
3924    history_events: Arc<Vec<HistoryEvent>>,
3925    signal_events: Arc<Vec<QuerySignal>>,
3926}
3927
3928impl QueryContext {
3929    /// The normalized argument list used to start the workflow.
3930    pub fn workflow_input(&self) -> &Value {
3931        &self.workflow_input
3932    }
3933
3934    /// The lossless fixed Avro Value argument list used to start the workflow.
3935    pub fn workflow_input_avro_value(&self) -> &AvroValue {
3936        &self.workflow_input_avro_value
3937    }
3938
3939    /// The immutable committed history used for this query snapshot.
3940    pub fn history_events(&self) -> &[HistoryEvent] {
3941        self.history_events.as_slice()
3942    }
3943
3944    /// All decoded signals in committed workflow order.
3945    pub fn signal_events(&self) -> &[QuerySignal] {
3946        self.signal_events.as_slice()
3947    }
3948
3949    /// Decoded argument lists for each committed signal with `signal_name`.
3950    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
3951        self.signal_events
3952            .iter()
3953            .filter(|signal| signal.name == signal_name)
3954            .map(|signal| signal.arguments.clone())
3955            .collect()
3956    }
3957
3958    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
3959    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
3960        self.signal_events
3961            .iter()
3962            .filter(|signal| signal.name == signal_name)
3963            .map(|signal| signal.avro_arguments.clone())
3964            .collect()
3965    }
3966}
3967
3968#[derive(Clone, Debug, Deserialize)]
3969pub struct ActivityHeartbeatResponse {
3970    #[serde(default)]
3971    pub cancel_requested: bool,
3972    #[serde(default)]
3973    pub heartbeat_recorded: bool,
3974    #[serde(default)]
3975    pub can_continue: Option<bool>,
3976    #[serde(default)]
3977    pub reason: Option<String>,
3978    #[serde(default)]
3979    pub run_closed_reason: Option<String>,
3980    #[serde(default)]
3981    pub run_closed_at: Option<String>,
3982    #[serde(default)]
3983    pub lease_expires_at: Option<String>,
3984    #[serde(default)]
3985    pub last_heartbeat_at: Option<String>,
3986}
3987
3988impl ActivityHeartbeatResponse {
3989    /// Whether the activity should stop instead of attempting completion.
3990    pub fn should_stop(&self) -> bool {
3991        self.cancel_requested || self.can_continue == Some(false)
3992    }
3993}
3994
3995fn default_payload_codec() -> String {
3996    DEFAULT_CODEC.to_string()
3997}
3998
3999fn default_workflow_task_attempt() -> u64 {
4000    1
4001}
4002
4003fn default_attempt_number() -> u64 {
4004    1
4005}
4006
4007type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4008type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
4009type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
4010type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
4011type ReplayedWorkflowHandler =
4012    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
4013type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4014type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
4015type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
4016type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4017type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
4018type ReplayedQueryHandler = Arc<
4019    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
4020        + Send
4021        + Sync,
4022>;
4023type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
4024
4025struct ReplayedWorkflowInvocation {
4026    future: WorkflowFuture,
4027    snapshot: WorkflowStateSnapshot,
4028}
4029
4030#[derive(Clone)]
4031struct RegisteredWorkflow {
4032    execute: WorkflowHandler,
4033    replay: Option<ReplayedWorkflowHandler>,
4034    state_type: Option<TypeId>,
4035}
4036
4037#[derive(Clone)]
4038enum RegisteredQuery {
4039    Snapshot(QueryHandler),
4040    Replayed {
4041        state_type: TypeId,
4042        handler: ReplayedQueryHandler,
4043    },
4044}
4045
4046#[derive(Clone, Debug)]
4047pub struct WorkerHeartbeatObservation {
4048    pub worker_id: String,
4049    pub task_queue: String,
4050    pub acknowledged_at_unix_millis: u64,
4051    pub acknowledgement: Value,
4052}
4053
4054/// Bounded retry policy for worker poll acquisition and worker heartbeats.
4055///
4056/// Expected empty long polls are normal successful responses. Transport
4057/// failures, HTTP 408/429 responses, and server errors are retried with capped
4058/// exponential backoff. Authentication, protocol, codec, and handler failures
4059/// are never retried by the worker.
4060#[derive(Clone, Copy, Debug)]
4061pub struct WorkerRetryPolicy {
4062    /// Number of retries after the initial request fails.
4063    pub max_retries: usize,
4064    /// Delay before the first retry.
4065    pub initial_backoff: Duration,
4066    /// Maximum delay between retries.
4067    pub max_backoff: Duration,
4068}
4069
4070impl Default for WorkerRetryPolicy {
4071    fn default() -> Self {
4072        Self {
4073            max_retries: 5,
4074            initial_backoff: Duration::from_millis(100),
4075            max_backoff: Duration::from_secs(5),
4076        }
4077    }
4078}
4079
4080#[derive(Clone, Copy, Debug, PartialEq, Eq)]
4081enum ManagedPollOutcome {
4082    Idle,
4083    Handled,
4084    Stop,
4085}
4086
4087#[derive(Clone)]
4088pub struct Worker {
4089    client: Client,
4090    worker_id: String,
4091    task_queue: String,
4092    workflows: HashMap<String, RegisteredWorkflow>,
4093    activities: HashMap<String, ActivityHandler>,
4094    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
4095    updates: HashMap<String, HashMap<String, UpdateHandler>>,
4096    max_concurrent_workflow_tasks: usize,
4097    max_concurrent_activity_tasks: usize,
4098    poll_timeout: Duration,
4099    heartbeat_interval: Duration,
4100    retry_policy: WorkerRetryPolicy,
4101    heartbeat_observer: Option<WorkerHeartbeatObserver>,
4102}
4103
4104impl Worker {
4105    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
4106        Self {
4107            client,
4108            worker_id: default_worker_id(),
4109            task_queue: task_queue.into(),
4110            workflows: HashMap::new(),
4111            activities: HashMap::new(),
4112            queries: HashMap::new(),
4113            updates: HashMap::new(),
4114            max_concurrent_workflow_tasks: 10,
4115            max_concurrent_activity_tasks: 10,
4116            poll_timeout: Duration::from_secs(30),
4117            heartbeat_interval: Duration::from_secs(60),
4118            retry_policy: WorkerRetryPolicy::default(),
4119            heartbeat_observer: None,
4120        }
4121    }
4122
4123    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
4124        self.worker_id = worker_id.into();
4125        self
4126    }
4127
4128    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
4129        self.poll_timeout = timeout;
4130        self
4131    }
4132
4133    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
4134        self.heartbeat_interval = interval;
4135        self
4136    }
4137
4138    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
4139    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
4140        self.retry_policy = policy;
4141        self
4142    }
4143
4144    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
4145    where
4146        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
4147    {
4148        self.heartbeat_observer = Some(Arc::new(observer));
4149        self
4150    }
4151
4152    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
4153        self.max_concurrent_workflow_tasks = count.max(1);
4154        self
4155    }
4156
4157    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
4158        self.max_concurrent_activity_tasks = count.max(1);
4159        self
4160    }
4161
4162    /// Register a workflow handler.
4163    ///
4164    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
4165    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
4166    /// while acquiring or decoding a worker task remain worker-operation
4167    /// failures and do not get converted into workflow outcomes.
4168    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
4169    where
4170        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
4171        Fut: Future<Output = Result<Value>> + Send + 'static,
4172    {
4173        let handler = Arc::new(handler);
4174        self.workflows.insert(
4175            workflow_type.into(),
4176            RegisteredWorkflow {
4177                execute: Arc::new(move |ctx, input| {
4178                    let handler = Arc::clone(&handler);
4179                    Box::pin(async move {
4180                        let result = handler(ctx, input.into_json()?).await?;
4181                        AvroValue::from_serialize(&result)
4182                    })
4183                }),
4184                replay: None,
4185                state_type: None,
4186            },
4187        );
4188    }
4189
4190    /// Register a workflow on the lossless fixed Avro Value surface.
4191    pub fn register_workflow_avro_value<F, Fut>(
4192        &mut self,
4193        workflow_type: impl Into<String>,
4194        handler: F,
4195    ) where
4196        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
4197        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4198    {
4199        self.workflows.insert(
4200            workflow_type.into(),
4201            RegisteredWorkflow {
4202                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
4203                replay: None,
4204                state_type: None,
4205            },
4206        );
4207    }
4208
4209    /// Register a workflow whose typed instance state can be reconstructed for queries.
4210    ///
4211    /// `state_factory` creates a fresh instance for every normal workflow task and
4212    /// query replay. The workflow handler is the single source of truth for state
4213    /// transitions: it updates [`WorkflowInstance`] after activities and signals
4214    /// resolve. Query replay runs this same handler over committed history and
4215    /// discards any commands it would emit.
4216    pub fn register_replayed_workflow<S, Factory, F, Fut>(
4217        &mut self,
4218        workflow_type: impl Into<String>,
4219        state_factory: Factory,
4220        handler: F,
4221    ) where
4222        S: Clone + Send + Sync + 'static,
4223        Factory: Fn() -> S + Send + Sync + 'static,
4224        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4225        Fut: Future<Output = Result<Value>> + Send + 'static,
4226    {
4227        let state_factory = Arc::new(state_factory);
4228        let handler = Arc::new(handler);
4229
4230        let execute_factory = Arc::clone(&state_factory);
4231        let execute_handler = Arc::clone(&handler);
4232        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4233            let state = WorkflowInstance::new(execute_factory());
4234            let handler = Arc::clone(&execute_handler);
4235            Box::pin(async move {
4236                let result = handler(ctx, input.into_json()?, state).await?;
4237                AvroValue::from_serialize(&result)
4238            }) as WorkflowFuture
4239        });
4240
4241        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4242            let state = WorkflowInstance::new(state_factory());
4243            let snapshot_state = state.clone();
4244            let snapshot: WorkflowStateSnapshot =
4245                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4246            let replay_handler = Arc::clone(&handler);
4247            let future = async move {
4248                let result = replay_handler(ctx, input.into_json()?, state).await?;
4249                AvroValue::from_serialize(&result)
4250            };
4251            ReplayedWorkflowInvocation {
4252                future: Box::pin(future),
4253                snapshot,
4254            }
4255        });
4256
4257        self.workflows.insert(
4258            workflow_type.into(),
4259            RegisteredWorkflow {
4260                execute,
4261                replay: Some(replay),
4262                state_type: Some(TypeId::of::<S>()),
4263            },
4264        );
4265    }
4266
4267    /// Register a replayable workflow on the lossless fixed Avro Value surface.
4268    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
4269        &mut self,
4270        workflow_type: impl Into<String>,
4271        state_factory: Factory,
4272        handler: F,
4273    ) where
4274        S: Clone + Send + Sync + 'static,
4275        Factory: Fn() -> S + Send + Sync + 'static,
4276        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
4277        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4278    {
4279        let state_factory = Arc::new(state_factory);
4280        let handler = Arc::new(handler);
4281
4282        let execute_factory = Arc::clone(&state_factory);
4283        let execute_handler = Arc::clone(&handler);
4284        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4285            let state = WorkflowInstance::new(execute_factory());
4286            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
4287        });
4288
4289        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
4290            let state = WorkflowInstance::new(state_factory());
4291            let snapshot_state = state.clone();
4292            let snapshot: WorkflowStateSnapshot =
4293                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
4294            ReplayedWorkflowInvocation {
4295                future: Box::pin(handler(ctx, input, state)),
4296                snapshot,
4297            }
4298        });
4299
4300        self.workflows.insert(
4301            workflow_type.into(),
4302            RegisteredWorkflow {
4303                execute,
4304                replay: Some(replay),
4305                state_type: Some(TypeId::of::<S>()),
4306            },
4307        );
4308    }
4309
4310    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
4311    where
4312        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
4313        Fut: Future<Output = Result<Value>> + Send + 'static,
4314    {
4315        let handler = Arc::new(handler);
4316        self.activities.insert(
4317            activity_type.into(),
4318            Arc::new(move |ctx, args| {
4319                let handler = Arc::clone(&handler);
4320                Box::pin(async move {
4321                    let result = handler(ctx, args.into_json()?).await?;
4322                    AvroValue::from_serialize(&result)
4323                })
4324            }),
4325        );
4326    }
4327
4328    /// Register an activity on the lossless fixed Avro Value surface.
4329    pub fn register_activity_avro_value<F, Fut>(
4330        &mut self,
4331        activity_type: impl Into<String>,
4332        handler: F,
4333    ) where
4334        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
4335        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4336    {
4337        self.activities.insert(
4338            activity_type.into(),
4339            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4340        );
4341    }
4342
4343    /// Register a named, read-only query handler for a workflow type.
4344    ///
4345    /// The workflow type must also be registered with [`Worker::register_workflow`]
4346    /// before the worker runs. The handler receives only an immutable committed
4347    /// state snapshot and normalized query arguments.
4348    pub fn register_query<F, Fut>(
4349        &mut self,
4350        workflow_type: impl Into<String>,
4351        query_name: impl Into<String>,
4352        handler: F,
4353    ) where
4354        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4355        Fut: Future<Output = Result<Value>> + Send + 'static,
4356    {
4357        let handler = Arc::new(handler);
4358        self.queries
4359            .entry(workflow_type.into())
4360            .or_default()
4361            .insert(
4362                query_name.into(),
4363                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
4364                    let handler = Arc::clone(&handler);
4365                    Box::pin(async move {
4366                        let result = handler(ctx, args.into_json()?).await?;
4367                        AvroValue::from_serialize(&result)
4368                    })
4369                })),
4370            );
4371    }
4372
4373    /// Register a query handler on the lossless fixed Avro Value surface.
4374    pub fn register_query_avro_value<F, Fut>(
4375        &mut self,
4376        workflow_type: impl Into<String>,
4377        query_name: impl Into<String>,
4378        handler: F,
4379    ) where
4380        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4381        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4382    {
4383        self.queries
4384            .entry(workflow_type.into())
4385            .or_default()
4386            .insert(
4387                query_name.into(),
4388                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
4389            );
4390    }
4391
4392    /// Register a named query against deterministically replayed instance state.
4393    ///
4394    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
4395    /// same state type `S`. The handler receives an immutable, detached state
4396    /// clone, so successful and failed queries cannot affect workflow execution
4397    /// or the state reconstructed by a later query.
4398    pub fn register_replayed_query<S, F, Fut>(
4399        &mut self,
4400        workflow_type: impl Into<String>,
4401        query_name: impl Into<String>,
4402        handler: F,
4403    ) where
4404        S: Clone + Send + Sync + 'static,
4405        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
4406        Fut: Future<Output = Result<Value>> + Send + 'static,
4407    {
4408        let handler = Arc::new(handler);
4409        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4410            let state = state.downcast::<S>().map_err(|_| {
4411                "registered query state type does not match the replayed workflow state".to_string()
4412            })?;
4413            let handler = Arc::clone(&handler);
4414            Ok(Box::pin(async move {
4415                let result = handler(ctx, state, args.into_json()?).await?;
4416                AvroValue::from_serialize(&result)
4417            }))
4418        });
4419
4420        self.queries
4421            .entry(workflow_type.into())
4422            .or_default()
4423            .insert(
4424                query_name.into(),
4425                RegisteredQuery::Replayed {
4426                    state_type: TypeId::of::<S>(),
4427                    handler: erased_handler,
4428                },
4429            );
4430    }
4431
4432    /// Register a replayed-state query on the lossless fixed Avro Value surface.
4433    pub fn register_replayed_query_avro_value<S, F, Fut>(
4434        &mut self,
4435        workflow_type: impl Into<String>,
4436        query_name: impl Into<String>,
4437        handler: F,
4438    ) where
4439        S: Clone + Send + Sync + 'static,
4440        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
4441        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4442    {
4443        let handler = Arc::new(handler);
4444        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
4445            let state = state.downcast::<S>().map_err(|_| {
4446                "registered query state type does not match the replayed workflow state".to_string()
4447            })?;
4448            Ok(Box::pin(handler(ctx, state, args)))
4449        });
4450
4451        self.queries
4452            .entry(workflow_type.into())
4453            .or_default()
4454            .insert(
4455                query_name.into(),
4456                RegisteredQuery::Replayed {
4457                    state_type: TypeId::of::<S>(),
4458                    handler: erased_handler,
4459                },
4460            );
4461    }
4462
4463    /// Register a synchronous workflow update handler.
4464    pub fn register_update<F, Fut>(
4465        &mut self,
4466        workflow_type: impl Into<String>,
4467        update_name: impl Into<String>,
4468        handler: F,
4469    ) where
4470        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
4471        Fut: Future<Output = Result<Value>> + Send + 'static,
4472    {
4473        let handler = Arc::new(handler);
4474        self.updates
4475            .entry(workflow_type.into())
4476            .or_default()
4477            .insert(
4478                update_name.into(),
4479                Arc::new(move |ctx, args| {
4480                    let handler = Arc::clone(&handler);
4481                    Box::pin(async move {
4482                        let result = handler(ctx, args.into_json()?).await?;
4483                        AvroValue::from_serialize(&result)
4484                    })
4485                }),
4486            );
4487    }
4488
4489    /// Register an update handler on the lossless fixed Avro Value surface.
4490    pub fn register_update_avro_value<F, Fut>(
4491        &mut self,
4492        workflow_type: impl Into<String>,
4493        update_name: impl Into<String>,
4494        handler: F,
4495    ) where
4496        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
4497        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
4498    {
4499        self.updates
4500            .entry(workflow_type.into())
4501            .or_default()
4502            .insert(
4503                update_name.into(),
4504                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
4505            );
4506    }
4507
4508    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
4509        let mut command_contracts = serde_json::Map::new();
4510        for workflow_type in self.workflows.keys() {
4511            let mut queries = self
4512                .queries
4513                .get(workflow_type)
4514                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4515                .unwrap_or_default();
4516            queries.sort();
4517            let mut updates = self
4518                .updates
4519                .get(workflow_type)
4520                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
4521                .unwrap_or_default();
4522            updates.sort();
4523            if !queries.is_empty() || !updates.is_empty() {
4524                command_contracts.insert(
4525                    workflow_type.clone(),
4526                    json!({
4527                        "queries": queries,
4528                        "updates": updates,
4529                        "update_validators": [],
4530                    }),
4531                );
4532            }
4533        }
4534
4535        self.client
4536            .register_worker_with_command_contracts(
4537                &self.worker_id,
4538                &self.task_queue,
4539                self.workflows.keys().cloned().collect(),
4540                self.activities.keys().cloned().collect(),
4541                self.max_concurrent_workflow_tasks,
4542                self.max_concurrent_activity_tasks,
4543                [
4544                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
4545                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
4546                ]
4547                .into_iter()
4548                .flatten()
4549                .collect(),
4550                Value::Object(command_contracts),
4551            )
4552            .await
4553    }
4554
4555    /// Run until shutdown or a terminal worker error occurs.
4556    ///
4557    /// Empty long-poll expirations do not stop the worker. Retryable poll and
4558    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
4559    /// authentication, protocol, and other non-retryable failures are returned.
4560    pub async fn run(&self) -> Result<()> {
4561        self.run_until(std::future::pending::<()>()).await
4562    }
4563
4564    /// Run until `shutdown` resolves or a terminal worker error occurs.
4565    ///
4566    /// This has the same liveness and terminal-error contract as [`Worker::run`].
4567    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
4568    where
4569        F: Future<Output = ()>,
4570    {
4571        let registration = self.register().await?;
4572        if !registration.registered {
4573            return Err(Error::WorkerLoop(format!(
4574                "worker registration for {:?} was not accepted",
4575                self.worker_id
4576            )));
4577        }
4578        let registered_worker_id = registration.worker_id.clone();
4579        let primary = self.run_registered_until(shutdown, registration).await;
4580        let deregistration = self
4581            .client
4582            .deregister_worker_registration(&registered_worker_id)
4583            .await;
4584
4585        match (primary, deregistration) {
4586            (Ok(()), Ok(_)) => Ok(()),
4587            (Ok(()), Err(deregistration)) => Err(deregistration),
4588            (Err(primary), Ok(_)) => Err(primary),
4589            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
4590                primary: Box::new(primary),
4591                deregistration: Box::new(deregistration),
4592            }),
4593        }
4594    }
4595
4596    async fn run_registered_until<F>(
4597        &self,
4598        shutdown: F,
4599        registration: RegisterWorkerResponse,
4600    ) -> Result<()>
4601    where
4602        F: Future<Output = ()>,
4603    {
4604        let heartbeat_interval = Duration::from_secs(
4605            registration
4606                .heartbeat_interval_seconds
4607                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
4608        );
4609        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
4610        // from the completion of the preceding attempt, including its bounded
4611        // retries. A fixed-epoch interval can leave an already-due tick queued
4612        // while an acknowledgement is slow, producing a catch-up heartbeat as
4613        // soon as that request completes.
4614        let heartbeat = tokio::time::sleep(Duration::ZERO);
4615        tokio::pin!(heartbeat);
4616        tokio::pin!(shutdown);
4617        let stop = Arc::new(AtomicBool::new(false));
4618        // Poll responses may already have leased server-side work by the time
4619        // they become ready, so each poller owns its responses through
4620        // completion or failure instead of racing raw polls in this select.
4621        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
4622            let worker = self.clone();
4623            let stop = Arc::clone(&stop);
4624            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
4625        });
4626        let mut activity_poller = (!self.activities.is_empty()).then(|| {
4627            let worker = self.clone();
4628            let stop = Arc::clone(&stop);
4629            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
4630        });
4631        let mut query_poller = (!self.queries.is_empty()).then(|| {
4632            let worker = self.clone();
4633            let stop = Arc::clone(&stop);
4634            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
4635        });
4636
4637        loop {
4638            tokio::select! {
4639                _ = &mut shutdown => {
4640                    stop.store(true, Ordering::SeqCst);
4641                    break;
4642                }
4643                _ = &mut heartbeat => {
4644                    let result = self.retry_worker_operation(|| {
4645                        self.client.heartbeat_worker(
4646                            &self.worker_id,
4647                            self.max_concurrent_workflow_tasks,
4648                            self.max_concurrent_activity_tasks,
4649                        )
4650                    }).await;
4651                    heartbeat
4652                        .as_mut()
4653                        .reset(tokio::time::Instant::now() + heartbeat_interval);
4654                    match result {
4655                        Ok(acknowledgement) => {
4656                            if let Some(observer) = &self.heartbeat_observer {
4657                                observer(&WorkerHeartbeatObservation {
4658                                    worker_id: self.worker_id.clone(),
4659                                    task_queue: self.task_queue.clone(),
4660                                    acknowledged_at_unix_millis: SystemTime::now()
4661                                        .duration_since(UNIX_EPOCH)
4662                                        .unwrap_or_default()
4663                                        .as_millis()
4664                                        .min(u64::MAX as u128)
4665                                        as u64,
4666                                    acknowledgement,
4667                                });
4668                            }
4669                        }
4670                        Err(error) => {
4671                            stop.store(true, Ordering::SeqCst);
4672                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
4673                            return Err(error);
4674                        }
4675                    }
4676                }
4677                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
4678                    workflow_poller = None;
4679                    let stopped_by_server = stop.load(Ordering::SeqCst);
4680                    stop.store(true, Ordering::SeqCst);
4681                    let poller_result = optional_poller_result("workflow", result);
4682                    let join_result =
4683                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4684                    poller_result?;
4685                    join_result?;
4686                    if stopped_by_server {
4687                        return Ok(());
4688                    }
4689                    return Err(Error::WorkerLoop(
4690                        "workflow poller stopped unexpectedly".to_string(),
4691                    ));
4692                }
4693                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
4694                    activity_poller = None;
4695                    let stopped_by_server = stop.load(Ordering::SeqCst);
4696                    stop.store(true, Ordering::SeqCst);
4697                    let poller_result = optional_poller_result("activity", result);
4698                    let join_result =
4699                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4700                    poller_result?;
4701                    join_result?;
4702                    if stopped_by_server {
4703                        return Ok(());
4704                    }
4705                    return Err(Error::WorkerLoop(
4706                        "activity poller stopped unexpectedly".to_string(),
4707                    ));
4708                }
4709                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
4710                    query_poller = None;
4711                    let stopped_by_server = stop.load(Ordering::SeqCst);
4712                    stop.store(true, Ordering::SeqCst);
4713                    let poller_result = optional_poller_result("query", result);
4714                    let join_result =
4715                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
4716                    poller_result?;
4717                    join_result?;
4718                    if stopped_by_server {
4719                        return Ok(());
4720                    }
4721                    return Err(Error::WorkerLoop(
4722                        "query poller stopped unexpectedly".to_string(),
4723                    ));
4724                }
4725            }
4726        }
4727
4728        join_pollers(
4729            workflow_poller.take(),
4730            activity_poller.take(),
4731            query_poller.take(),
4732        )
4733        .await
4734    }
4735
4736    /// Poll and settle at most one task from each enabled task family.
4737    ///
4738    /// A workflow may reach its server-enforced run deadline while this worker
4739    /// holds a task. When the completion endpoint authoritatively rejects that
4740    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
4741    /// terminal `run_status=failed`, the workflow tick is considered settled:
4742    /// the late command was not recorded and cannot replace the terminal run.
4743    /// Every other completion rejection remains an error. This worker-level
4744    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
4745    /// only bounds how long a client waits for a result.
4746    ///
4747    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
4748    /// the original [`Error::Http`] status and response body.
4749    pub async fn run_once(&self) -> Result<usize> {
4750        let mut handled = 0;
4751        match self.poll_workflow_once().await? {
4752            ManagedPollOutcome::Handled => handled += 1,
4753            ManagedPollOutcome::Stop => return Ok(handled),
4754            ManagedPollOutcome::Idle => {}
4755        }
4756        match self.poll_activity_once().await? {
4757            ManagedPollOutcome::Handled => handled += 1,
4758            ManagedPollOutcome::Stop => return Ok(handled),
4759            ManagedPollOutcome::Idle => {}
4760        }
4761        if !self.queries.is_empty() {
4762            match self.poll_query_once().await? {
4763                ManagedPollOutcome::Handled => handled += 1,
4764                ManagedPollOutcome::Stop => return Ok(handled),
4765                ManagedPollOutcome::Idle => {}
4766            }
4767        }
4768        Ok(handled)
4769    }
4770
4771    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
4772        let poll_request_id = unique_request_id("rust-workflow-poll");
4773        let response = self
4774            .retry_worker_operation(|| {
4775                self.client.poll_workflow_task_response_with_request_id(
4776                    &self.worker_id,
4777                    &self.task_queue,
4778                    self.poll_timeout,
4779                    &poll_request_id,
4780                    0,
4781                )
4782            })
4783            .await?;
4784        if response.outcome().should_stop() {
4785            return Ok(ManagedPollOutcome::Stop);
4786        }
4787        let Some(task) = response.task else {
4788            return Ok(ManagedPollOutcome::Idle);
4789        };
4790
4791        let task_id = task.task_id.clone();
4792        let attempt = task.workflow_task_attempt;
4793        let run_id = task.run_id.clone();
4794        let lease_owner = task
4795            .lease_owner
4796            .clone()
4797            .unwrap_or_else(|| self.worker_id.clone());
4798
4799        match self.execute_workflow_task(task) {
4800            Ok(commands) if commands.is_empty() => {
4801                // A replay can consume a recorded pending durable command
4802                // without producing a new command. The standalone protocol
4803                // acknowledges that state through the typed waiting outcome;
4804                // an empty completion is rejected by servers that require at
4805                // least one executable command.
4806                self.client
4807                    .fail_workflow_task_with_type(
4808                        &task_id,
4809                        &lease_owner,
4810                        attempt,
4811                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
4812                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
4813                    )
4814                    .await?;
4815            }
4816            Ok(commands) => {
4817                let completion = self
4818                    .client
4819                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
4820                    .await;
4821                if let Err(error) = completion {
4822                    if !workflow_task_completion_is_terminal_timeout(
4823                        &error,
4824                        &task_id,
4825                        attempt,
4826                        run_id.as_deref(),
4827                    ) {
4828                        return Err(error);
4829                    }
4830                }
4831            }
4832            Err(error) => {
4833                self.client
4834                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
4835                    .await?;
4836            }
4837        }
4838
4839        Ok(ManagedPollOutcome::Handled)
4840    }
4841
4842    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4843        while !stop.load(Ordering::SeqCst) {
4844            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
4845                stop.store(true, Ordering::SeqCst);
4846                break;
4847            }
4848        }
4849
4850        Ok(())
4851    }
4852
4853    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
4854        let poll_request_id = unique_request_id("rust-activity-poll");
4855        let response = self
4856            .retry_worker_operation(|| {
4857                self.client.poll_activity_task_response_with_request_id(
4858                    &self.worker_id,
4859                    &self.task_queue,
4860                    self.poll_timeout,
4861                    &poll_request_id,
4862                    0,
4863                )
4864            })
4865            .await?;
4866        if response.outcome().should_stop() {
4867            return Ok(ManagedPollOutcome::Stop);
4868        }
4869        let Some(task) = response.task else {
4870            return Ok(ManagedPollOutcome::Idle);
4871        };
4872
4873        let task_id = task.task_id.clone();
4874        let attempt_id = task
4875            .activity_attempt_id
4876            .clone()
4877            .or(task.attempt_id.clone())
4878            .unwrap_or_default();
4879        let lease_owner = task
4880            .lease_owner
4881            .clone()
4882            .unwrap_or_else(|| self.worker_id.clone());
4883        let codec = task.payload_codec.clone();
4884        let result = self.execute_activity_task(task).await;
4885        match result {
4886            Ok(value) => {
4887                let completion = self
4888                    .client
4889                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
4890                    .await;
4891                if let Err(error) = completion {
4892                    if !activity_task_rejection_is_final(&error) {
4893                        return Err(error);
4894                    }
4895                }
4896            }
4897            Err(error) => {
4898                let failure = self
4899                    .client
4900                    .fail_activity_task(
4901                        &task_id,
4902                        &attempt_id,
4903                        &lease_owner,
4904                        error.to_string(),
4905                        false,
4906                    )
4907                    .await;
4908                if let Err(error) = failure {
4909                    if !activity_task_rejection_is_final(&error) {
4910                        return Err(error);
4911                    }
4912                }
4913            }
4914        }
4915
4916        Ok(ManagedPollOutcome::Handled)
4917    }
4918
4919    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
4920        while !stop.load(Ordering::SeqCst) {
4921            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
4922                stop.store(true, Ordering::SeqCst);
4923                break;
4924            }
4925        }
4926
4927        Ok(())
4928    }
4929
4930    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
4931        let poll_request_id = unique_request_id("rust-query-poll");
4932        let response = self
4933            .retry_worker_operation(|| {
4934                self.client.poll_query_task_response_with_request_id(
4935                    &self.worker_id,
4936                    &self.task_queue,
4937                    self.poll_timeout,
4938                    &poll_request_id,
4939                    0,
4940                )
4941            })
4942            .await?;
4943        if response.outcome().should_stop() {
4944            return Ok(ManagedPollOutcome::Stop);
4945        }
4946        let Some(task) = response.task else {
4947            return Ok(ManagedPollOutcome::Idle);
4948        };
4949
4950        let query_task_id = task.query_task_id.clone();
4951        let attempt = task.query_task_attempt;
4952        let lease_owner = task
4953            .lease_owner
4954            .clone()
4955            .unwrap_or_else(|| self.worker_id.clone());
4956        let codec = task.payload_codec.clone();
4957
4958        match self.execute_query_task(task).await {
4959            Ok(value) => {
4960                let result_envelope = match encode_typed_envelope(&value, &codec) {
4961                    Ok(result_envelope) => result_envelope,
4962                    Err(error) => {
4963                        let failure = self
4964                            .client
4965                            .fail_query_task(
4966                                &query_task_id,
4967                                &lease_owner,
4968                                attempt,
4969                                error.to_string(),
4970                                "query_result_encode_failed",
4971                                "QueryResultEncodeFailed",
4972                            )
4973                            .await;
4974                        if let Err(error) = failure {
4975                            if !query_task_rejection_is_final(&error) {
4976                                return Err(error);
4977                            }
4978                        }
4979                        return Ok(ManagedPollOutcome::Handled);
4980                    }
4981                };
4982
4983                if let Err(error) = self
4984                    .client
4985                    .complete_query_task_with_envelope(
4986                        &query_task_id,
4987                        &lease_owner,
4988                        attempt,
4989                        value.clone().into_json()?,
4990                        result_envelope,
4991                    )
4992                    .await
4993                {
4994                    if !query_task_rejection_is_final(&error) {
4995                        return Err(error);
4996                    }
4997                }
4998            }
4999            Err(failure) => {
5000                let result = self
5001                    .client
5002                    .fail_query_task(
5003                        &query_task_id,
5004                        &lease_owner,
5005                        attempt,
5006                        failure.message,
5007                        failure.reason,
5008                        failure.failure_type,
5009                    )
5010                    .await;
5011                if let Err(error) = result {
5012                    if !query_task_rejection_is_final(&error) {
5013                        return Err(error);
5014                    }
5015                }
5016            }
5017        }
5018
5019        Ok(ManagedPollOutcome::Handled)
5020    }
5021
5022    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
5023        while !stop.load(Ordering::SeqCst) {
5024            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
5025                stop.store(true, Ordering::SeqCst);
5026                break;
5027            }
5028        }
5029
5030        Ok(())
5031    }
5032
5033    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
5034    where
5035        F: FnMut() -> Fut,
5036        Fut: Future<Output = Result<T>>,
5037    {
5038        let mut retries = 0;
5039
5040        loop {
5041            match operation().await {
5042                Err(error)
5043                    if worker_operation_is_retryable(&error)
5044                        && retries < self.retry_policy.max_retries =>
5045                {
5046                    retries += 1;
5047                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
5048                }
5049                result => return result,
5050            }
5051        }
5052    }
5053
5054    async fn execute_query_task(
5055        &self,
5056        mut task: QueryTask,
5057    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
5058        if !matches!(task.payload_codec.as_str(), DEFAULT_CODEC | JSON_CODEC) {
5059            return Err(QueryTaskExecutionFailure::new(
5060                "query_payload_decode_failed",
5061                format!(
5062                    "cannot decode query payload with unsupported codec {:?}",
5063                    task.payload_codec
5064                ),
5065                "QueryPayloadDecodeFailed",
5066            ));
5067        }
5068
5069        if !self.workflows.contains_key(&task.workflow_type) {
5070            return Err(QueryTaskExecutionFailure::new(
5071                "query_workflow_type_not_registered",
5072                format!("no workflow registered for type {:?}", task.workflow_type),
5073                "WorkflowTypeNotRegistered",
5074            ));
5075        }
5076
5077        let Some(handlers) = self.queries.get(&task.workflow_type) else {
5078            return Err(QueryTaskExecutionFailure::new(
5079                "query_handler_unavailable",
5080                format!(
5081                    "query handlers are unavailable for workflow type {:?}",
5082                    task.workflow_type
5083                ),
5084                "QueryHandlerUnavailable",
5085            ));
5086        };
5087        let Some(query) = handlers.get(&task.query_name) else {
5088            return Err(QueryTaskExecutionFailure::new(
5089                "rejected_unknown_query",
5090                format!("unknown query {:?}", task.query_name),
5091                "QueryFailed",
5092            ));
5093        };
5094
5095        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
5096            .map_err(|error| {
5097            QueryTaskExecutionFailure::new(
5098                "query_payload_decode_failed",
5099                format!("cannot decode query arguments: {error}"),
5100                "QueryPayloadDecodeFailed",
5101            )
5102        })?;
5103        let workflow_input_typed =
5104            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
5105                .map_err(|error| {
5106                    QueryTaskExecutionFailure::new(
5107                        "query_workflow_state_unavailable",
5108                        format!("cannot decode workflow start input: {error}"),
5109                        "QueryWorkflowStateUnavailable",
5110                    )
5111                })?;
5112        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
5113            QueryTaskExecutionFailure::new(
5114                "query_workflow_state_unavailable",
5115                format!("cannot project workflow start input: {error}"),
5116                "QueryWorkflowStateUnavailable",
5117            )
5118        })?;
5119        hydrate_query_history_from_export(&mut task).map_err(|error| {
5120            QueryTaskExecutionFailure::new(
5121                "query_workflow_state_unavailable",
5122                format!("cannot restore query history snapshot: {error}"),
5123                "QueryWorkflowStateUnavailable",
5124            )
5125        })?;
5126        enrich_query_history_from_export(&mut task).map_err(|error| {
5127            QueryTaskExecutionFailure::new(
5128                "query_workflow_state_unavailable",
5129                format!("cannot restore compact query history payloads: {error}"),
5130                "QueryWorkflowStateUnavailable",
5131            )
5132        })?;
5133        let signal_events = query_signal_events(&task).map_err(|error| {
5134            QueryTaskExecutionFailure::new(
5135                "query_workflow_state_unavailable",
5136                format!("cannot decode committed workflow signals: {error}"),
5137                "QueryWorkflowStateUnavailable",
5138            )
5139        })?;
5140        let history_events = Arc::new(std::mem::take(&mut task.history_events));
5141        let context = QueryContext {
5142            workflow_id: task.workflow_id,
5143            run_id: task.run_id,
5144            workflow_type: task.workflow_type.clone(),
5145            run_status: task.run_status,
5146            workflow_input,
5147            workflow_input_avro_value: workflow_input_typed.clone(),
5148            history_events: Arc::clone(&history_events),
5149            signal_events: Arc::new(signal_events),
5150        };
5151
5152        let future = match query {
5153            RegisteredQuery::Snapshot(handler) => handler(context, args),
5154            RegisteredQuery::Replayed {
5155                state_type,
5156                handler,
5157            } => {
5158                let workflow = self
5159                    .workflows
5160                    .get(&task.workflow_type)
5161                    .expect("workflow registration was checked above");
5162                if workflow.state_type != Some(*state_type) {
5163                    return Err(QueryTaskExecutionFailure::new(
5164                        "query_workflow_state_unavailable",
5165                        "replayed query state type does not match its workflow registration",
5166                        "QueryWorkflowStateUnavailable",
5167                    ));
5168                }
5169                let replay = workflow.replay.as_ref().ok_or_else(|| {
5170                    QueryTaskExecutionFailure::new(
5171                        "query_workflow_state_unavailable",
5172                        format!(
5173                            "workflow type {:?} is not registered for instance-state replay",
5174                            task.workflow_type
5175                        ),
5176                        "QueryWorkflowStateUnavailable",
5177                    )
5178                })?;
5179                let workflow_state = Arc::new(Mutex::new(
5180                    WorkflowState::new_with_identity(
5181                        history_events.as_ref().clone(),
5182                        context.workflow_id.clone(),
5183                        context.run_id.clone(),
5184                        self.task_queue.clone(),
5185                        task.payload_codec,
5186                        None,
5187                    )
5188                    .map_err(|error| {
5189                        QueryTaskExecutionFailure::new(
5190                            "query_workflow_state_unavailable",
5191                            format!("workflow replay failed before query: {error}"),
5192                            "QueryWorkflowStateUnavailable",
5193                        )
5194                    })?,
5195                ));
5196                let workflow_context = WorkflowContext {
5197                    state: workflow_state,
5198                };
5199                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
5200                let mut cx = TaskContext::from_waker(noop_waker_ref());
5201                match invocation.future.as_mut().poll(&mut cx) {
5202                    Poll::Ready(Ok(_)) => {
5203                        workflow_context
5204                            .ensure_history_consumed()
5205                            .map_err(|error| {
5206                                QueryTaskExecutionFailure::new(
5207                                    "query_workflow_state_unavailable",
5208                                    format!("workflow replay failed before query: {error}"),
5209                                    "QueryWorkflowStateUnavailable",
5210                                )
5211                            })?;
5212                    }
5213                    Poll::Ready(Err(error)) => {
5214                        return Err(QueryTaskExecutionFailure::new(
5215                            "query_workflow_state_unavailable",
5216                            format!("workflow replay failed before query: {error}"),
5217                            "QueryWorkflowStateUnavailable",
5218                        ));
5219                    }
5220                    Poll::Pending => {
5221                        let commands = workflow_context.take_commands().map_err(|error| {
5222                            QueryTaskExecutionFailure::new(
5223                                "query_workflow_state_unavailable",
5224                                format!("workflow replay failed before query: {error}"),
5225                                "QueryWorkflowStateUnavailable",
5226                            )
5227                        })?;
5228                        if commands.is_empty()
5229                            && !workflow_context
5230                                .matched_recorded_pending()
5231                                .map_err(|error| {
5232                                    QueryTaskExecutionFailure::new(
5233                                        "query_workflow_state_unavailable",
5234                                        format!("workflow replay failed before query: {error}"),
5235                                        "QueryWorkflowStateUnavailable",
5236                                    )
5237                                })?
5238                        {
5239                            return Err(QueryTaskExecutionFailure::new(
5240                                "query_workflow_state_unavailable",
5241                                "workflow replay yielded without a durable command",
5242                                "QueryWorkflowStateUnavailable",
5243                            ));
5244                        }
5245                    }
5246                }
5247                let state = (invocation.snapshot)().map_err(|error| {
5248                    QueryTaskExecutionFailure::new(
5249                        "query_workflow_state_unavailable",
5250                        format!("cannot snapshot replayed workflow state: {error}"),
5251                        "QueryWorkflowStateUnavailable",
5252                    )
5253                })?;
5254                handler(context, state, args).map_err(|message| {
5255                    QueryTaskExecutionFailure::new(
5256                        "query_workflow_state_unavailable",
5257                        message,
5258                        "QueryWorkflowStateUnavailable",
5259                    )
5260                })?
5261            }
5262        };
5263
5264        future.await.map_err(|error| {
5265            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
5266        })
5267    }
5268
5269    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
5270        if let Some(update_id) = task
5271            .workflow_update_id
5272            .as_deref()
5273            .filter(|update_id| !update_id.is_empty())
5274        {
5275            return self.execute_update_task(&task, update_id);
5276        }
5277
5278        let workflow = self
5279            .workflows
5280            .get(&task.workflow_type)
5281            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
5282        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5283        let resume_signal = decode_resume_signal(&task)?;
5284        let history_budget = WorkflowHistoryBudget {
5285            event_count: task
5286                .total_history_events
5287                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
5288            size_bytes: task.history_size_bytes,
5289            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
5290            pressure: task.history_budget_pressure.clone(),
5291        };
5292        let mut workflow_state = WorkflowState::new_with_identity(
5293            task.history_events,
5294            task.workflow_id,
5295            task.run_id,
5296            self.task_queue.clone(),
5297            task.payload_codec.clone(),
5298            resume_signal,
5299        )?;
5300        workflow_state.history_budget = history_budget;
5301        let state = Arc::new(Mutex::new(workflow_state));
5302        let ctx = WorkflowContext { state };
5303        let mut future = (workflow.execute)(ctx.clone(), input);
5304        let mut cx = TaskContext::from_waker(noop_waker_ref());
5305
5306        match future.as_mut().poll(&mut cx) {
5307            Poll::Ready(Ok(result)) => {
5308                ctx.ensure_history_consumed()?;
5309                let result = encode_typed_envelope(&result, &task.payload_codec)?;
5310                let mut commands = ctx.take_commands()?;
5311                commands.push(json!({
5312                    "type": "complete_workflow",
5313                    "result": result
5314                }));
5315                Ok(commands)
5316            }
5317            Poll::Ready(Err(error)) => {
5318                if let Error::ContinueAsNew(request) = error {
5319                    let mut commands = ctx.take_commands()?;
5320                    if let Some(command) = ctx.continue_as_new_command(request)? {
5321                        commands.push(command);
5322                    }
5323                    ctx.ensure_history_consumed()?;
5324                    return Ok(commands);
5325                }
5326                // A handler error must not hide a committed durable command that
5327                // upgraded workflow code no longer consumes.
5328                ctx.ensure_history_consumed()?;
5329                if workflow_task_integrity_error(&error) {
5330                    // Replay and protocol failures describe the workflow-task
5331                    // decision itself. Do not let commands queued earlier in
5332                    // this uncommitted decision escape alongside a terminal
5333                    // workflow failure.
5334                    return Err(error);
5335                }
5336                let mut commands = ctx.take_commands()?;
5337                commands.push(workflow_failure_command(&error));
5338                Ok(commands)
5339            }
5340            Poll::Pending => {
5341                let commands = ctx.take_commands()?;
5342                if commands.is_empty() && !ctx.matched_recorded_pending()? {
5343                    Err(Error::WorkflowYieldedWithoutCommand)
5344                } else {
5345                    Ok(commands)
5346                }
5347            }
5348        }
5349    }
5350
5351    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
5352        if !self.workflows.contains_key(&task.workflow_type) {
5353            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
5354        }
5355
5356        let accepted = task.history_events.iter().rev().find_map(|event| {
5357            (event.event_type == "UpdateAccepted"
5358                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
5359            .then_some(&event.payload)
5360        });
5361        let update_name = accepted
5362            .and_then(|payload| payload.get("update_name"))
5363            .and_then(Value::as_str)
5364            .or(task.update_name.as_deref())
5365            .unwrap_or_default();
5366        let Some(handler) = self
5367            .updates
5368            .get(&task.workflow_type)
5369            .and_then(|handlers| handlers.get(update_name))
5370        else {
5371            return Ok(vec![json!({
5372                "type": "fail_update",
5373                "update_id": update_id,
5374                "message": format!(
5375                    "no update handler is registered for {}.{update_name}",
5376                    task.workflow_type
5377                ),
5378                "exception_type": "UnknownUpdate",
5379                "non_retryable": true,
5380            })]);
5381        };
5382        let arguments = accepted
5383            .and_then(|payload| payload.get("arguments"))
5384            .or(task.arguments.as_ref());
5385        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
5386        let context = QueryContext {
5387            workflow_id: task.workflow_id.clone(),
5388            run_id: task.run_id.clone(),
5389            workflow_type: task.workflow_type.clone(),
5390            run_status: Some("running".to_string()),
5391            workflow_input: Value::Null,
5392            workflow_input_avro_value: AvroValue::Null,
5393            history_events: Arc::new(task.history_events.clone()),
5394            signal_events: Arc::new(Vec::new()),
5395        };
5396        let mut future = handler(context, arguments);
5397        let mut cx = TaskContext::from_waker(noop_waker_ref());
5398
5399        match future.as_mut().poll(&mut cx) {
5400            Poll::Ready(Ok(result)) => Ok(vec![json!({
5401                "type": "complete_update",
5402                "update_id": update_id,
5403                "result": encode_typed_envelope(&result, &task.payload_codec)?,
5404            })]),
5405            Poll::Ready(Err(error)) => Ok(vec![json!({
5406                "type": "fail_update",
5407                "update_id": update_id,
5408                "message": error.to_string(),
5409                "exception_type": "UpdateFailed",
5410                "non_retryable": true,
5411            })]),
5412            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
5413        }
5414    }
5415
5416    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
5417        let handler = self
5418            .activities
5419            .get(&task.activity_type)
5420            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
5421        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
5422        let attempt_id = task
5423            .activity_attempt_id
5424            .clone()
5425            .or(task.attempt_id.clone())
5426            .unwrap_or_default();
5427        let lease_owner = task
5428            .lease_owner
5429            .clone()
5430            .unwrap_or_else(|| self.worker_id.clone());
5431        let ctx = ActivityContext {
5432            client: self.client.clone(),
5433            task_id: task.task_id,
5434            activity_attempt_id: attempt_id,
5435            lease_owner,
5436            activity_type: task.activity_type,
5437            attempt_number: task.attempt_number,
5438            task_queue: self.task_queue.clone(),
5439            worker_id: self.worker_id.clone(),
5440        };
5441
5442        handler(ctx, args).await
5443    }
5444}
5445
5446fn poller_result(
5447    kind: &str,
5448    result: std::result::Result<Result<()>, tokio::task::JoinError>,
5449) -> Result<()> {
5450    match result {
5451        Ok(result) => result,
5452        Err(error) => Err(Error::WorkerLoop(format!(
5453            "{kind} poller join error: {error}"
5454        ))),
5455    }
5456}
5457
5458fn optional_poller_result(
5459    kind: &str,
5460    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
5461) -> Result<()> {
5462    match result {
5463        Some(result) => poller_result(kind, result),
5464        None => Ok(()),
5465    }
5466}
5467
5468async fn join_pollers(
5469    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5470    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5471    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
5472) -> Result<()> {
5473    let mut first_error = None;
5474
5475    if let Some(handle) = workflow_poller {
5476        if let Err(error) = poller_result("workflow", handle.await) {
5477            first_error.get_or_insert(error);
5478        }
5479    }
5480
5481    if let Some(handle) = activity_poller {
5482        if let Err(error) = poller_result("activity", handle.await) {
5483            first_error.get_or_insert(error);
5484        }
5485    }
5486
5487    if let Some(handle) = query_poller {
5488        if let Err(error) = poller_result("query", handle.await) {
5489            first_error.get_or_insert(error);
5490        }
5491    }
5492
5493    if let Some(error) = first_error {
5494        Err(error)
5495    } else {
5496        Ok(())
5497    }
5498}
5499
5500fn default_worker_id() -> String {
5501    let millis = SystemTime::now()
5502        .duration_since(UNIX_EPOCH)
5503        .unwrap_or_default()
5504        .as_millis();
5505    format!("rust-worker-{}-{millis}", std::process::id())
5506}
5507
5508fn percent_encode_path_segment(segment: &str) -> String {
5509    const HEX: &[u8; 16] = b"0123456789ABCDEF";
5510    let mut encoded = String::with_capacity(segment.len());
5511
5512    for byte in segment.bytes() {
5513        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
5514            encoded.push(char::from(byte));
5515        } else {
5516            encoded.push('%');
5517            encoded.push(char::from(HEX[(byte >> 4) as usize]));
5518            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
5519        }
5520    }
5521
5522    encoded
5523}
5524
5525fn unique_request_id(prefix: &str) -> String {
5526    let nanos = SystemTime::now()
5527        .duration_since(UNIX_EPOCH)
5528        .unwrap_or_default()
5529        .as_nanos();
5530    format!("{prefix}-{}-{nanos}", std::process::id())
5531}
5532
5533#[derive(Debug)]
5534struct QueryTaskExecutionFailure {
5535    reason: String,
5536    message: String,
5537    failure_type: String,
5538}
5539
5540impl QueryTaskExecutionFailure {
5541    fn new(
5542        reason: impl Into<String>,
5543        message: impl Into<String>,
5544        failure_type: impl Into<String>,
5545    ) -> Self {
5546        Self {
5547            reason: reason.into(),
5548            message: message.into(),
5549            failure_type: failure_type.into(),
5550        }
5551    }
5552}
5553
5554/// Typed local state owned by one deterministic workflow invocation.
5555///
5556/// Use [`WorkflowInstance::update`] for the same state transitions during
5557/// ordinary execution and replay. A replayed query receives a detached
5558/// immutable `Arc<S>` rather than this mutation-capable handle.
5559#[derive(Clone, Debug)]
5560pub struct WorkflowInstance<S> {
5561    state: Arc<Mutex<S>>,
5562}
5563
5564impl<S> WorkflowInstance<S> {
5565    fn new(state: S) -> Self {
5566        Self {
5567            state: Arc::new(Mutex::new(state)),
5568        }
5569    }
5570
5571    /// Read the current workflow-instance state without changing it.
5572    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
5573        let state = self
5574            .state
5575            .lock()
5576            .map_err(|_| Error::WorkflowStatePoisoned)?;
5577        Ok(reader(&state))
5578    }
5579
5580    /// Apply one deterministic workflow-instance state transition.
5581    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
5582        let mut state = self
5583            .state
5584            .lock()
5585            .map_err(|_| Error::WorkflowStatePoisoned)?;
5586        Ok(transition(&mut state))
5587    }
5588}
5589
5590impl<S: Clone> WorkflowInstance<S> {
5591    fn snapshot(&self) -> Result<S> {
5592        self.read(Clone::clone)
5593    }
5594}
5595
5596#[derive(Clone, Debug)]
5597pub struct WorkflowContext {
5598    state: Arc<Mutex<WorkflowState>>,
5599}
5600
5601impl WorkflowContext {
5602    /// Identity of the parent workflow currently being replayed.
5603    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
5604        let state = self
5605            .state
5606            .lock()
5607            .map_err(|_| Error::WorkflowStatePoisoned)?;
5608        Ok(WorkflowIdentity {
5609            workflow_id: state.workflow_id.clone(),
5610            run_id: state.run_id.clone(),
5611        })
5612    }
5613
5614    /// Return the server-published history budget for this workflow task.
5615    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
5616        let state = self
5617            .state
5618            .lock()
5619            .map_err(|_| Error::WorkflowStatePoisoned)?;
5620        Ok(state.history_budget.clone())
5621    }
5622
5623    /// Continue this workflow instance as a fresh run with replacement arguments.
5624    ///
5625    /// Return this value directly from the workflow handler. The worker converts
5626    /// it to the terminal protocol command only after replay has consumed every
5627    /// recorded durable command.
5628    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
5629        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
5630    }
5631
5632    /// Continue as new with optional workflow-type and task-queue overrides.
5633    pub fn continue_as_new_with_options<T: Serialize>(
5634        &self,
5635        options: ContinueAsNewOptions,
5636        args: T,
5637    ) -> Result<Value> {
5638        options.validate()?;
5639        Err(Error::ContinueAsNew(ContinueAsNewRequest {
5640            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
5641            options,
5642        }))
5643    }
5644
5645    pub fn activity<T: Serialize>(
5646        &self,
5647        activity_type: impl Into<String>,
5648        args: T,
5649    ) -> ActivityCall {
5650        self.activity_with_options(activity_type, ActivityOptions::new(), args)
5651    }
5652
5653    pub fn activity_on_queue<T, Q>(
5654        &self,
5655        activity_type: impl Into<String>,
5656        task_queue: Option<Q>,
5657        args: T,
5658    ) -> ActivityCall
5659    where
5660        T: Serialize,
5661        Q: Into<String>,
5662    {
5663        let mut options = ActivityOptions::new();
5664        options.task_queue = task_queue.map(Into::into);
5665        self.activity_with_options(activity_type, options, args)
5666    }
5667
5668    /// Schedule one durable activity with retry, routing, and timeout options.
5669    ///
5670    /// Options are validated before the command is emitted. Once the command is
5671    /// recorded, replay consumes the same activity lifecycle at this command
5672    /// position and never emits a duplicate schedule.
5673    ///
5674    /// ```no_run
5675    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
5676    /// # use std::time::Duration;
5677    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
5678    /// let result = ctx
5679    ///     .activity_with_options(
5680    ///         "charge-card",
5681    ///         ActivityOptions::new()
5682    ///             .task_queue("payments")
5683    ///             .retry_policy(
5684    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
5685    ///                     Duration::from_secs(1),
5686    ///                     2,
5687    ///                     Some(Duration::from_secs(30)),
5688    ///                 ),
5689    ///             )
5690    ///             .start_to_close_timeout(Duration::from_secs(60))
5691    ///             .schedule_to_close_timeout(Duration::from_secs(180))
5692    ///             .heartbeat_timeout(Duration::from_secs(15)),
5693    ///         json!([{"order_id": "order-42"}]),
5694    ///     )
5695    ///     .await;
5696    /// match result {
5697    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
5698    ///         "reason": failure.reason,
5699    ///         "timeout_kind": failure.timeout_kind,
5700    ///     })),
5701    ///     other => other,
5702    /// }
5703    /// # }
5704    /// ```
5705    pub fn activity_with_options<T: Serialize>(
5706        &self,
5707        activity_type: impl Into<String>,
5708        options: ActivityOptions,
5709        args: T,
5710    ) -> ActivityCall {
5711        ActivityCall {
5712            ctx: self.clone(),
5713            activity_type: activity_type.into(),
5714            options,
5715            args: Some(AvroValue::from_serialize(&args)),
5716            scheduled: false,
5717        }
5718    }
5719
5720    pub async fn activity_avro_value<T: Serialize>(
5721        &self,
5722        activity_type: impl Into<String>,
5723        args: T,
5724    ) -> Result<AvroValue> {
5725        let mut call = self.activity(activity_type, args);
5726        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5727    }
5728
5729    pub async fn activity_avro_value_with_options<T: Serialize>(
5730        &self,
5731        activity_type: impl Into<String>,
5732        options: ActivityOptions,
5733        args: T,
5734    ) -> Result<AvroValue> {
5735        let mut call = self.activity_with_options(activity_type, options, args);
5736        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5737    }
5738
5739    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
5740        SignalCall {
5741            ctx: self.clone(),
5742            signal_name: signal_name.into(),
5743            opened_wait: false,
5744            matched_pending: false,
5745        }
5746    }
5747
5748    pub async fn wait_signal_avro_value(
5749        &self,
5750        signal_name: impl Into<String>,
5751    ) -> Result<Vec<AvroValue>> {
5752        let mut call = self.wait_signal(signal_name);
5753        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
5754    }
5755
5756    /// Wait for server-backed durable time without blocking the worker executor.
5757    ///
5758    /// Polling this future emits one `start_timer` command and yields. The
5759    /// server records the deadline, so neither worker nor server restarts reset
5760    /// the wait. Replay resolves the future only from a `TimerScheduled` and
5761    /// `TimerFired` pair at the same position in the shared durable-command
5762    /// stream, with matching sequence, timer identity, and delay. Sub-second
5763    /// durations round up because protocol deadlines use whole seconds.
5764    ///
5765    /// ```no_run
5766    /// # use durable_workflow::{json, Client, Worker};
5767    /// # use std::time::Duration;
5768    /// # fn configure(client: Client) {
5769    /// let mut worker = Worker::new(client, "rust-workers");
5770    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
5771    ///     ctx.sleep(Duration::from_secs(5)).await?;
5772    ///     Ok(json!({"status": "timer fired"}))
5773    /// });
5774    /// # }
5775    /// ```
5776    pub fn sleep(&self, duration: Duration) -> TimerCall {
5777        let delay_seconds = duration
5778            .as_secs()
5779            .checked_add(u64::from(duration.subsec_nanos() > 0));
5780        TimerCall {
5781            ctx: self.clone(),
5782            delay_seconds,
5783            scheduled: false,
5784            matched_pending: false,
5785        }
5786    }
5787
5788    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
5789    pub fn start_timer(&self, duration: Duration) -> TimerCall {
5790        self.sleep(duration)
5791    }
5792
5793    /// Evaluate a non-deterministic callback once and durably record its typed value.
5794    ///
5795    /// On replay the callback is not invoked: the value is decoded from the
5796    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
5797    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
5798    /// small values that must remain fixed for the lifetime of a workflow run.
5799    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
5800    where
5801        T: Serialize + DeserializeOwned,
5802        F: FnOnce() -> T,
5803    {
5804        {
5805            let mut state = self
5806                .state
5807                .lock()
5808                .map_err(|_| Error::WorkflowStatePoisoned)?;
5809            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5810                return match recorded {
5811                    RecordedCommand::SideEffect { sequence, value } => {
5812                        state.command_cursor += 1;
5813                        value.deserialize().map_err(|error| {
5814                            Error::NonDeterministicReplay(ReplayFailure::new(
5815                                "side_effect_type_mismatch",
5816                                Some(sequence),
5817                                Some(std::any::type_name::<T>().to_string()),
5818                                Some(error.to_string()),
5819                                "recorded side-effect value is incompatible with the requested Rust type",
5820                            ))
5821                        })
5822                    }
5823                    other => Err(command_mismatch(&other, "side effect")),
5824                };
5825            }
5826        }
5827
5828        let value = callback();
5829        let avro_value = AvroValue::from_serialize(&value)?;
5830        let mut state = self
5831            .state
5832            .lock()
5833            .map_err(|_| Error::WorkflowStatePoisoned)?;
5834        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
5835        state.commands.push(json!({
5836            "type": "record_side_effect",
5837            "result": result,
5838        }));
5839        Ok(value)
5840    }
5841
5842    /// Record or replay a lossless fixed Avro Value side effect.
5843    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
5844    where
5845        F: FnOnce() -> AvroValue,
5846    {
5847        {
5848            let mut state = self
5849                .state
5850                .lock()
5851                .map_err(|_| Error::WorkflowStatePoisoned)?;
5852            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5853                return match recorded {
5854                    RecordedCommand::SideEffect { value, .. } => {
5855                        state.command_cursor += 1;
5856                        Ok(value)
5857                    }
5858                    other => Err(command_mismatch(&other, "side effect")),
5859                };
5860            }
5861        }
5862
5863        let value = callback();
5864        let mut state = self
5865            .state
5866            .lock()
5867            .map_err(|_| Error::WorkflowStatePoisoned)?;
5868        let result = encode_typed_envelope(&value, &state.payload_codec)?;
5869        state.commands.push(json!({
5870            "type": "record_side_effect",
5871            "result": result,
5872        }));
5873        Ok(value)
5874    }
5875
5876    /// Record a UUIDv4 once and return the same UUID on every replay.
5877    pub fn uuid_v4(&self) -> Result<Uuid> {
5878        self.side_effect(Uuid::new_v4)
5879    }
5880
5881    /// Select the newest supported version for a change, or replay the version
5882    /// already committed for that stable change ID.
5883    pub fn get_version(
5884        &self,
5885        change_id: impl Into<String>,
5886        min_supported: i32,
5887        max_supported: i32,
5888    ) -> Result<i32> {
5889        let change_id = change_id.into();
5890        if change_id.trim().is_empty() {
5891            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5892                "version_change_id_invalid",
5893                None,
5894                Some("non-empty change ID".to_string()),
5895                Some(change_id),
5896                "version markers require a stable non-empty change ID",
5897            )));
5898        }
5899        if min_supported > max_supported {
5900            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5901                "version_range_invalid",
5902                None,
5903                Some("min_supported <= max_supported".to_string()),
5904                Some(format!("{min_supported}..={max_supported}")),
5905                "version marker supported range is invalid",
5906            )));
5907        }
5908
5909        let mut state = self
5910            .state
5911            .lock()
5912            .map_err(|_| Error::WorkflowStatePoisoned)?;
5913        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
5914            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
5915            return Ok(version);
5916        }
5917
5918        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
5919            return match recorded {
5920                RecordedCommand::VersionMarker {
5921                    sequence,
5922                    change_id: recorded_change_id,
5923                    version,
5924                    ..
5925                } => {
5926                    if recorded_change_id != change_id {
5927                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
5928                            "version_change_id_mismatch",
5929                            Some(sequence),
5930                            Some(recorded_change_id),
5931                            Some(change_id),
5932                            "recorded version marker change ID differs from current workflow code",
5933                        )));
5934                    }
5935                    ensure_version_supported(
5936                        &change_id,
5937                        version,
5938                        min_supported,
5939                        max_supported,
5940                        sequence,
5941                    )?;
5942                    state.command_cursor += 1;
5943                    state.version_markers.insert(change_id, (version, sequence));
5944                    Ok(version)
5945                }
5946                other => Err(command_mismatch(
5947                    &other,
5948                    format!("version marker:{change_id}"),
5949                )),
5950            };
5951        }
5952
5953        let version = max_supported;
5954        state.commands.push(json!({
5955            "type": "record_version_marker",
5956            "change_id": change_id,
5957            "version": version,
5958            "min_supported": min_supported,
5959            "max_supported": max_supported,
5960        }));
5961        // Sequence numbers are assigned by the server. Zero identifies a marker
5962        // selected in this uncommitted decision batch for duplicate-call checks.
5963        state.version_markers.insert(change_id, (version, 0));
5964        Ok(version)
5965    }
5966
5967    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
5968    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
5969        Ok(self.get_version(change_id, -1, 1)? == 1)
5970    }
5971
5972    /// Keep a patch marker in history after the legacy branch has been removed.
5973    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
5974        self.get_version(change_id, -1, 1).map(|_| ())
5975    }
5976
5977    /// Start a named durable child on an explicit queue and await its result.
5978    ///
5979    /// The command is recorded in the parent's sequence-ordered durable command
5980    /// stream. Replay keeps a scheduled child pending without emitting another
5981    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
5982    /// values preserve the history payload codec and include both sides of the
5983    /// durable relationship; failures are returned as
5984    /// [`Error::ChildWorkflowFailed`].
5985    ///
5986    /// ```no_run
5987    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
5988    /// # fn configure(client: Client) {
5989    /// let mut worker = Worker::new(client, "parent-workers");
5990    /// worker.register_workflow("order-parent", |ctx, _input| async move {
5991    ///     let child = ctx
5992    ///         .start_child_workflow(
5993    ///             "fulfil-order",
5994    ///             ChildWorkflowOptions::new("fulfilment-workers")
5995    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
5996    ///             json!([{"order_id": "order-42"}]),
5997    ///         )
5998    ///         .await?;
5999    ///     Ok(child.result)
6000    /// });
6001    /// # }
6002    /// ```
6003    pub fn start_child_workflow<T: Serialize>(
6004        &self,
6005        workflow_type: impl Into<String>,
6006        options: ChildWorkflowOptions,
6007        args: T,
6008    ) -> ChildWorkflowCall {
6009        ChildWorkflowCall {
6010            ctx: self.clone(),
6011            workflow_type: workflow_type.into(),
6012            options,
6013            args: Some(AvroValue::from_serialize(&args)),
6014            scheduled: false,
6015            matched_pending: false,
6016        }
6017    }
6018
6019    pub async fn start_child_workflow_avro_value<T: Serialize>(
6020        &self,
6021        workflow_type: impl Into<String>,
6022        options: ChildWorkflowOptions,
6023        args: T,
6024    ) -> Result<ChildWorkflowAvroResult> {
6025        let mut call = self.start_child_workflow(workflow_type, options, args);
6026        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
6027    }
6028
6029    fn take_commands(&self) -> Result<Vec<Value>> {
6030        let mut state = self
6031            .state
6032            .lock()
6033            .map_err(|_| Error::WorkflowStatePoisoned)?;
6034        Ok(std::mem::take(&mut state.commands))
6035    }
6036
6037    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
6038        let mut state = self
6039            .state
6040            .lock()
6041            .map_err(|_| Error::WorkflowStatePoisoned)?;
6042
6043        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6044            return Err(command_mismatch(&recorded, "continue as new"));
6045        }
6046        if state.recorded_continue_as_new_sequence.is_some() {
6047            state.continue_as_new_consumed = true;
6048            return Ok(None);
6049        }
6050
6051        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
6052        let mut command = serde_json::Map::from_iter([
6053            ("type".to_string(), json!("continue_as_new")),
6054            ("arguments".to_string(), arguments),
6055            ("queue".to_string(), json!(state.task_queue.clone())),
6056        ]);
6057        if let Some(workflow_type) = request.options.workflow_type {
6058            command.insert("workflow_type".to_string(), json!(workflow_type));
6059        }
6060        if let Some(task_queue) = request.options.task_queue {
6061            command.insert("queue".to_string(), json!(task_queue));
6062        }
6063        Ok(Some(Value::Object(command)))
6064    }
6065
6066    fn matched_recorded_pending(&self) -> Result<bool> {
6067        let state = self
6068            .state
6069            .lock()
6070            .map_err(|_| Error::WorkflowStatePoisoned)?;
6071        Ok(state.matched_recorded_pending)
6072    }
6073
6074    fn ensure_history_consumed(&self) -> Result<()> {
6075        let state = self
6076            .state
6077            .lock()
6078            .map_err(|_| Error::WorkflowStatePoisoned)?;
6079        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
6080            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6081                "recorded_commands_unconsumed",
6082                Some(command.sequence()),
6083                Some(command.shape().to_string()),
6084                Some("workflow completion".to_string()),
6085                "workflow completed before consuming all recorded durable commands",
6086            )));
6087        }
6088        if let Some(sequence) = state
6089            .recorded_continue_as_new_sequence
6090            .filter(|_| !state.continue_as_new_consumed)
6091        {
6092            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
6093                "recorded_continue_as_new_unconsumed",
6094                Some(sequence),
6095                Some("continue as new".to_string()),
6096                Some("workflow completion".to_string()),
6097                "workflow completed without consuming its recorded continue-as-new transition",
6098            )));
6099        }
6100        Ok(())
6101    }
6102}
6103
6104#[derive(Debug)]
6105struct WorkflowState {
6106    workflow_id: Option<String>,
6107    run_id: Option<String>,
6108    task_queue: String,
6109    payload_codec: String,
6110    history_budget: WorkflowHistoryBudget,
6111    resume_signal: Option<ResumeSignal>,
6112    recorded_commands: Vec<RecordedCommand>,
6113    recorded_continue_as_new_sequence: Option<u64>,
6114    continue_as_new_consumed: bool,
6115    command_cursor: usize,
6116    matched_recorded_pending: bool,
6117    version_markers: HashMap<String, (i32, u64)>,
6118    commands: Vec<Value>,
6119}
6120
6121impl WorkflowState {
6122    #[cfg(test)]
6123    fn new(
6124        history: Vec<HistoryEvent>,
6125        task_queue: String,
6126        payload_codec: String,
6127        resume_signal: Option<ResumeSignal>,
6128    ) -> Result<Self> {
6129        Self::new_with_identity(
6130            history,
6131            None,
6132            None,
6133            task_queue,
6134            payload_codec,
6135            resume_signal,
6136        )
6137    }
6138
6139    fn new_with_identity(
6140        history: Vec<HistoryEvent>,
6141        workflow_id: Option<String>,
6142        run_id: Option<String>,
6143        task_queue: String,
6144        payload_codec: String,
6145        resume_signal: Option<ResumeSignal>,
6146    ) -> Result<Self> {
6147        let recorded_commands = recorded_commands(
6148            &history,
6149            &payload_codec,
6150            WorkflowIdentity {
6151                workflow_id: workflow_id.clone(),
6152                run_id: run_id.clone(),
6153            },
6154        )?;
6155        let recorded_continue_as_new = history
6156            .iter()
6157            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
6158            .collect::<Vec<_>>();
6159        if recorded_continue_as_new.len() > 1 {
6160            return Err(invalid_recorded_history(
6161                "duplicate_continue_as_new_transition",
6162                recorded_continue_as_new
6163                    .last()
6164                    .and_then(|event| durable_event_sequence(event))
6165                    .unwrap_or(0),
6166                "one WorkflowContinuedAsNew event",
6167                &format!(
6168                    "{} WorkflowContinuedAsNew events",
6169                    recorded_continue_as_new.len()
6170                ),
6171                "workflow history records one continue-as-new transition more than once",
6172            ));
6173        }
6174        let recorded_continue_as_new_sequence = recorded_continue_as_new
6175            .first()
6176            .map(|event| {
6177                durable_event_sequence(event).ok_or_else(|| {
6178                    Error::NonDeterministicReplay(ReplayFailure::new(
6179                        "continue_as_new_sequence_missing",
6180                        None,
6181                        Some("recorded transition sequence".to_string()),
6182                        Some("missing sequence".to_string()),
6183                        "WorkflowContinuedAsNew history is missing its recorded sequence",
6184                    ))
6185                })
6186            })
6187            .transpose()?;
6188        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
6189        Ok(Self {
6190            workflow_id,
6191            run_id,
6192            task_queue,
6193            payload_codec,
6194            history_budget: WorkflowHistoryBudget {
6195                event_count,
6196                ..WorkflowHistoryBudget::default()
6197            },
6198            resume_signal,
6199            recorded_commands,
6200            recorded_continue_as_new_sequence,
6201            continue_as_new_consumed: false,
6202            command_cursor: 0,
6203            matched_recorded_pending: false,
6204            version_markers: HashMap::new(),
6205            commands: Vec::new(),
6206        })
6207    }
6208}
6209
6210#[derive(Clone, Debug)]
6211enum RecordedCommand {
6212    Activity {
6213        sequence: u64,
6214        activity_type: Option<String>,
6215        options: Option<RecordedActivityOptions>,
6216        outcome: Option<ActivityOutcome>,
6217    },
6218    Timer {
6219        sequence: u64,
6220        delay_seconds: u64,
6221        fired: bool,
6222    },
6223    ChildWorkflow {
6224        sequence: u64,
6225        workflow_type: Option<String>,
6226        outcome: Option<ChildWorkflowOutcome>,
6227    },
6228    SignalWait {
6229        sequence: u64,
6230        signal_name: String,
6231        value: Option<Vec<AvroValue>>,
6232    },
6233    SideEffect {
6234        sequence: u64,
6235        value: AvroValue,
6236    },
6237    VersionMarker {
6238        sequence: u64,
6239        change_id: String,
6240        version: i32,
6241    },
6242}
6243
6244#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6245struct RecordedActivityOptions {
6246    task_queue: RecordedSnapshotValue<Option<String>>,
6247    execution_mode: RecordedSnapshotValue<Option<String>>,
6248    retry_policy: ActivityRetrySnapshot,
6249}
6250
6251#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6252enum RecordedSnapshotValue<T> {
6253    /// Older history did not persist this field, so it cannot constrain replay.
6254    Unknown,
6255    Known(T),
6256}
6257
6258impl<T: PartialEq> RecordedSnapshotValue<T> {
6259    fn matches_current(&self, current: &Self) -> bool {
6260        match self {
6261            Self::Unknown => true,
6262            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
6263        }
6264    }
6265}
6266
6267#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
6268struct ActivityRetrySnapshot {
6269    snapshot_version: RecordedSnapshotValue<Option<u64>>,
6270    max_attempts: RecordedSnapshotValue<Option<u64>>,
6271    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
6272    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6273    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
6274    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
6275    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
6276    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
6277}
6278
6279impl ActivityRetrySnapshot {
6280    fn matches_current(&self, current: &Self) -> bool {
6281        self.snapshot_version
6282            .matches_current(&current.snapshot_version)
6283            && self.max_attempts.matches_current(&current.max_attempts)
6284            && self
6285                .backoff_seconds
6286                .matches_current(&current.backoff_seconds)
6287            && self
6288                .start_to_close_timeout
6289                .matches_current(&current.start_to_close_timeout)
6290            && self
6291                .schedule_to_start_timeout
6292                .matches_current(&current.schedule_to_start_timeout)
6293            && self
6294                .schedule_to_close_timeout
6295                .matches_current(&current.schedule_to_close_timeout)
6296            && self
6297                .heartbeat_timeout
6298                .matches_current(&current.heartbeat_timeout)
6299            && self
6300                .non_retryable_error_types
6301                .matches_current(&current.non_retryable_error_types)
6302    }
6303}
6304
6305fn recorded_optional_u64(
6306    object: Option<&serde_json::Map<String, Value>>,
6307    field: &str,
6308) -> RecordedSnapshotValue<Option<u64>> {
6309    match object.and_then(|object| object.get(field)) {
6310        None => RecordedSnapshotValue::Unknown,
6311        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6312        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
6313    }
6314}
6315
6316fn recorded_optional_string(
6317    object: &serde_json::Map<String, Value>,
6318    field: &str,
6319) -> RecordedSnapshotValue<Option<String>> {
6320    match object.get(field) {
6321        None => RecordedSnapshotValue::Unknown,
6322        Some(Value::Null) => RecordedSnapshotValue::Known(None),
6323        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
6324    }
6325}
6326
6327fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
6328    let policy = policy.and_then(Value::as_object);
6329    let backoff_seconds = policy
6330        .and_then(|policy| policy.get("backoff_seconds"))
6331        .and_then(Value::as_array)
6332        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6333        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
6334    let mut non_retryable_error_types = Vec::new();
6335    for error_type in policy
6336        .and_then(|policy| policy.get("non_retryable_error_types"))
6337        .and_then(Value::as_array)
6338        .into_iter()
6339        .flatten()
6340        .filter_map(Value::as_str)
6341        .map(str::trim)
6342        .filter(|error_type| !error_type.is_empty())
6343    {
6344        if !non_retryable_error_types
6345            .iter()
6346            .any(|recorded| recorded == error_type)
6347        {
6348            non_retryable_error_types.push(error_type.to_string());
6349        }
6350    }
6351
6352    ActivityRetrySnapshot {
6353        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
6354        max_attempts: recorded_optional_u64(policy, "max_attempts"),
6355        backoff_seconds,
6356        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
6357        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
6358        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
6359        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
6360        non_retryable_error_types: if policy
6361            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
6362        {
6363            RecordedSnapshotValue::Known(non_retryable_error_types)
6364        } else {
6365            RecordedSnapshotValue::Unknown
6366        },
6367    }
6368}
6369
6370fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
6371    let policy = options.retry_policy.as_ref();
6372    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
6373        Some(Value::Null) => None,
6374        Some(value) => value_as_u64(value),
6375        None => Some(1),
6376    };
6377    let backoff_seconds = policy
6378        .and_then(|policy| policy.get("backoff_seconds"))
6379        .and_then(Value::as_array)
6380        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
6381        .unwrap_or_default();
6382    let non_retryable_error_types = policy
6383        .and_then(|policy| policy.get("non_retryable_error_types"))
6384        .and_then(Value::as_array)
6385        .into_iter()
6386        .flatten()
6387        .filter_map(Value::as_str)
6388        .map(str::to_string)
6389        .collect();
6390
6391    ActivityRetrySnapshot {
6392        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
6393        max_attempts: RecordedSnapshotValue::Known(max_attempts),
6394        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
6395        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
6396        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
6397        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
6398        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
6399        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
6400    }
6401}
6402
6403fn activity_options_description(options: &RecordedActivityOptions) -> String {
6404    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
6405}
6406
6407impl RecordedCommand {
6408    fn sequence(&self) -> u64 {
6409        match self {
6410            Self::Activity { sequence, .. }
6411            | Self::Timer { sequence, .. }
6412            | Self::ChildWorkflow { sequence, .. }
6413            | Self::SignalWait { sequence, .. }
6414            | Self::SideEffect { sequence, .. }
6415            | Self::VersionMarker { sequence, .. } => *sequence,
6416        }
6417    }
6418
6419    fn shape(&self) -> &'static str {
6420        match self {
6421            Self::Activity { .. } => "activity",
6422            Self::Timer { .. } => "timer",
6423            Self::ChildWorkflow { .. } => "child workflow",
6424            Self::SignalWait { .. } => "signal wait",
6425            Self::SideEffect { .. } => "side effect",
6426            Self::VersionMarker { .. } => "version marker",
6427        }
6428    }
6429}
6430
6431fn ensure_version_supported(
6432    change_id: &str,
6433    version: i32,
6434    min_supported: i32,
6435    max_supported: i32,
6436    sequence: u64,
6437) -> Result<()> {
6438    if (min_supported..=max_supported).contains(&version) {
6439        return Ok(());
6440    }
6441    Err(Error::NonDeterministicReplay(ReplayFailure::new(
6442        "version_marker_incompatible_range",
6443        (sequence != 0).then_some(sequence),
6444        Some(format!("{min_supported}..={max_supported}")),
6445        Some(format!("{change_id}:{version}")),
6446        "recorded workflow version is outside the range supported by current code",
6447    )))
6448}
6449
6450#[derive(Clone, Debug)]
6451struct ResumeSignal {
6452    signal_name: String,
6453    arguments: Vec<AvroValue>,
6454}
6455
6456pub struct ActivityCall {
6457    ctx: WorkflowContext,
6458    activity_type: String,
6459    options: ActivityOptions,
6460    args: Option<Result<AvroValue>>,
6461    scheduled: bool,
6462}
6463
6464impl ActivityCall {
6465    fn poll_avro_value(
6466        mut self: Pin<&mut Self>,
6467        _cx: &mut TaskContext<'_>,
6468    ) -> Poll<Result<AvroValue>> {
6469        let ctx = self.ctx.clone();
6470        let mut state = match ctx.state.lock() {
6471            Ok(state) => state,
6472            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6473        };
6474
6475        if self.scheduled {
6476            return Poll::Pending;
6477        }
6478
6479        let options = match self.options.validate() {
6480            Ok(options) => options,
6481            Err(error) => {
6482                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
6483            }
6484        };
6485        let task_queue = options
6486            .task_queue
6487            .clone()
6488            .unwrap_or_else(|| state.task_queue.clone());
6489        let current_recorded_options = RecordedActivityOptions {
6490            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
6491            // Rust schedules ordinary durable activities. The server records a
6492            // non-null mode only for a specialized execution primitive.
6493            execution_mode: RecordedSnapshotValue::Known(None),
6494            retry_policy: current_activity_retry_snapshot(&options),
6495        };
6496
6497        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6498            let sequence = recorded.sequence();
6499            match recorded {
6500                RecordedCommand::Activity {
6501                    activity_type,
6502                    options: recorded_options,
6503                    outcome,
6504                    ..
6505                } => {
6506                    if let Some(recorded_type) = activity_type {
6507                        if recorded_type != self.activity_type {
6508                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6509                                ReplayFailure::new(
6510                                    "recorded_command_detail_mismatch",
6511                                    Some(sequence),
6512                                    Some(format!("activity:{recorded_type}")),
6513                                    Some(format!("activity:{}", self.activity_type)),
6514                                    "recorded activity type differs from the current workflow command",
6515                                ),
6516                            )));
6517                        }
6518                    }
6519                    if let Some(recorded_options) = recorded_options {
6520                        if !recorded_options
6521                            .task_queue
6522                            .matches_current(&current_recorded_options.task_queue)
6523                        {
6524                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6525                                ReplayFailure::new(
6526                                    "activity_task_queue_mismatch",
6527                                    Some(sequence),
6528                                    Some(activity_options_description(&recorded_options)),
6529                                    Some(activity_options_description(&current_recorded_options)),
6530                                    "recorded activity task queue differs from the current workflow command",
6531                                ),
6532                            )));
6533                        }
6534                        if !recorded_options
6535                            .execution_mode
6536                            .matches_current(&current_recorded_options.execution_mode)
6537                        {
6538                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6539                                ReplayFailure::new(
6540                                    "activity_execution_mode_mismatch",
6541                                    Some(sequence),
6542                                    Some(activity_options_description(&recorded_options)),
6543                                    Some(activity_options_description(&current_recorded_options)),
6544                                    "recorded activity execution mode differs from the current workflow command",
6545                                ),
6546                            )));
6547                        }
6548                        if !recorded_options
6549                            .retry_policy
6550                            .matches_current(&current_recorded_options.retry_policy)
6551                        {
6552                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6553                                ReplayFailure::new(
6554                                    "activity_retry_policy_mismatch",
6555                                    Some(sequence),
6556                                    Some(activity_options_description(&recorded_options)),
6557                                    Some(activity_options_description(&current_recorded_options)),
6558                                    "recorded activity retry policy differs from the current workflow command",
6559                                ),
6560                            )));
6561                        }
6562                    }
6563                    state.command_cursor += 1;
6564                    if let Some(outcome) = outcome {
6565                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
6566                    }
6567                    state.matched_recorded_pending = true;
6568                    self.scheduled = true;
6569                    return Poll::Pending;
6570                }
6571                other => {
6572                    return Poll::Ready(Err(command_mismatch(
6573                        &other,
6574                        format!("activity:{}", self.activity_type),
6575                    )));
6576                }
6577            }
6578        }
6579
6580        if !self.scheduled {
6581            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6582                Ok(args) => args,
6583                Err(error) => return Poll::Ready(Err(error)),
6584            };
6585            let arguments = normalize_avro_arguments(args);
6586            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
6587                Ok(envelope) => envelope,
6588                Err(error) => return Poll::Ready(Err(error)),
6589            };
6590
6591            let mut command = serde_json::Map::from_iter([
6592                ("type".to_string(), json!("schedule_activity")),
6593                (
6594                    "activity_type".to_string(),
6595                    json!(self.activity_type.clone()),
6596                ),
6597                ("queue".to_string(), json!(task_queue)),
6598                ("arguments".to_string(), envelope),
6599            ]);
6600            for (field, value) in [
6601                ("start_to_close_timeout", options.start_to_close_timeout),
6602                (
6603                    "schedule_to_start_timeout",
6604                    options.schedule_to_start_timeout,
6605                ),
6606                (
6607                    "schedule_to_close_timeout",
6608                    options.schedule_to_close_timeout,
6609                ),
6610                ("heartbeat_timeout", options.heartbeat_timeout),
6611            ] {
6612                if let Some(value) = value {
6613                    command.insert(field.to_string(), json!(value));
6614                }
6615            }
6616            if let Some(retry_policy) = options.retry_policy {
6617                command.insert("retry_policy".to_string(), retry_policy);
6618            }
6619            state.commands.push(Value::Object(command));
6620            self.scheduled = true;
6621        }
6622
6623        Poll::Pending
6624    }
6625}
6626
6627impl Future for ActivityCall {
6628    type Output = Result<Value>;
6629
6630    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6631        match self.poll_avro_value(cx) {
6632            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
6633            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6634            Poll::Pending => Poll::Pending,
6635        }
6636    }
6637}
6638
6639/// Future returned by [`WorkflowContext::sleep`].
6640pub struct TimerCall {
6641    ctx: WorkflowContext,
6642    delay_seconds: Option<u64>,
6643    scheduled: bool,
6644    matched_pending: bool,
6645}
6646
6647impl Future for TimerCall {
6648    type Output = Result<()>;
6649
6650    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6651        if self.matched_pending {
6652            return Poll::Pending;
6653        }
6654
6655        let ctx = self.ctx.clone();
6656        let Some(requested_delay) = self.delay_seconds else {
6657            return Poll::Ready(Err(Error::TimerDurationOverflow));
6658        };
6659        let mut state = match ctx.state.lock() {
6660            Ok(state) => state,
6661            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6662        };
6663
6664        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6665            match recorded {
6666                RecordedCommand::Timer {
6667                    sequence,
6668                    delay_seconds,
6669                    fired,
6670                    ..
6671                } => {
6672                    if delay_seconds != requested_delay {
6673                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6674                            ReplayFailure::new(
6675                                "timer_delay_mismatch",
6676                                Some(sequence),
6677                                Some(format!("timer:{delay_seconds}s")),
6678                                Some(format!("timer:{requested_delay}s")),
6679                                "recorded timer delay differs from the current workflow command",
6680                            ),
6681                        )));
6682                    }
6683                    state.command_cursor += 1;
6684                    if fired {
6685                        return Poll::Ready(Ok(()));
6686                    }
6687                    state.matched_recorded_pending = true;
6688                    self.scheduled = true;
6689                    self.matched_pending = true;
6690                    return Poll::Pending;
6691                }
6692                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
6693            }
6694        }
6695
6696        if !self.scheduled {
6697            state.commands.push(json!({
6698                "type": "start_timer",
6699                "delay_seconds": requested_delay,
6700            }));
6701            self.scheduled = true;
6702        }
6703
6704        Poll::Pending
6705    }
6706}
6707
6708/// Future returned by [`WorkflowContext::start_child_workflow`].
6709pub struct ChildWorkflowCall {
6710    ctx: WorkflowContext,
6711    workflow_type: String,
6712    options: ChildWorkflowOptions,
6713    args: Option<Result<AvroValue>>,
6714    scheduled: bool,
6715    matched_pending: bool,
6716}
6717
6718impl ChildWorkflowCall {
6719    fn poll_avro_value(
6720        mut self: Pin<&mut Self>,
6721        _cx: &mut TaskContext<'_>,
6722    ) -> Poll<Result<ChildWorkflowAvroResult>> {
6723        if self.matched_pending {
6724            return Poll::Pending;
6725        }
6726
6727        let ctx = self.ctx.clone();
6728        let mut state = match ctx.state.lock() {
6729            Ok(state) => state,
6730            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6731        };
6732
6733        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6734            let sequence = recorded.sequence();
6735            match recorded {
6736                RecordedCommand::ChildWorkflow {
6737                    workflow_type,
6738                    outcome,
6739                    ..
6740                } => {
6741                    if let Some(recorded_type) = workflow_type {
6742                        if recorded_type != self.workflow_type {
6743                            return Poll::Ready(Err(Error::NonDeterministicReplay(
6744                                ReplayFailure::new(
6745                                    "recorded_command_detail_mismatch",
6746                                    Some(sequence),
6747                                    Some(format!("child workflow:{recorded_type}")),
6748                                    Some(format!("child workflow:{}", self.workflow_type)),
6749                                    "recorded child workflow type differs from the current workflow command",
6750                                ),
6751                            )));
6752                        }
6753                    }
6754                    state.command_cursor += 1;
6755                    if let Some(outcome) = outcome {
6756                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
6757                    }
6758                    state.matched_recorded_pending = true;
6759                    self.scheduled = true;
6760                    self.matched_pending = true;
6761                    return Poll::Pending;
6762                }
6763                other => {
6764                    return Poll::Ready(Err(command_mismatch(
6765                        &other,
6766                        format!("child workflow:{}", self.workflow_type),
6767                    )));
6768                }
6769            }
6770        }
6771
6772        if !self.scheduled {
6773            if self.options.task_queue.trim().is_empty() {
6774                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6775                    "task_queue must not be empty".to_string(),
6776                )));
6777            }
6778            for (name, value) in [
6779                (
6780                    "execution_timeout_seconds",
6781                    self.options.execution_timeout_seconds,
6782                ),
6783                ("run_timeout_seconds", self.options.run_timeout_seconds),
6784            ] {
6785                if value == Some(0) {
6786                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
6787                        "{name} must be at least 1"
6788                    ))));
6789                }
6790            }
6791
6792            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
6793                Ok(args) => args,
6794                Err(error) => return Poll::Ready(Err(error)),
6795            };
6796            let arguments = match encode_typed_envelope(
6797                &normalize_avro_arguments(args),
6798                &state.payload_codec,
6799            ) {
6800                Ok(arguments) => arguments,
6801                Err(error) => return Poll::Ready(Err(error)),
6802            };
6803            let mut command = json!({
6804                "type": "start_child_workflow",
6805                "workflow_type": self.workflow_type,
6806                "queue": self.options.task_queue,
6807                "parent_close_policy": self.options.parent_close_policy.as_str(),
6808                "arguments": arguments,
6809            });
6810            let object = command
6811                .as_object_mut()
6812                .expect("child workflow command is always an object");
6813            if let Some(policy) = &self.options.retry_policy {
6814                let mut retry_policy = serde_json::Map::new();
6815                if let Some(max_attempts) = policy.max_attempts {
6816                    if max_attempts == 0 {
6817                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6818                            "retry_policy.max_attempts must be at least 1".to_string(),
6819                        )));
6820                    }
6821                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
6822                }
6823                if !policy.backoff_seconds.is_empty() {
6824                    retry_policy
6825                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
6826                }
6827                if !policy.non_retryable_error_types.is_empty() {
6828                    retry_policy.insert(
6829                        "non_retryable_error_types".to_string(),
6830                        json!(policy.non_retryable_error_types),
6831                    );
6832                }
6833                if retry_policy.is_empty() {
6834                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
6835                        "retry_policy must configure at least one field".to_string(),
6836                    )));
6837                }
6838                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
6839            }
6840            if let Some(seconds) = self.options.execution_timeout_seconds {
6841                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
6842            }
6843            if let Some(seconds) = self.options.run_timeout_seconds {
6844                object.insert("run_timeout_seconds".to_string(), json!(seconds));
6845            }
6846            state.commands.push(command);
6847            self.scheduled = true;
6848        }
6849
6850        Poll::Pending
6851    }
6852}
6853
6854impl Future for ChildWorkflowCall {
6855    type Output = Result<ChildWorkflowResult>;
6856
6857    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6858        match self.poll_avro_value(cx) {
6859            Poll::Ready(Ok(result)) => match result.result.into_json() {
6860                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
6861                    parent: result.parent,
6862                    child: result.child,
6863                    child_workflow_type: result.child_workflow_type,
6864                    result: projected,
6865                })),
6866                Err(error) => Poll::Ready(Err(error)),
6867            },
6868            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6869            Poll::Pending => Poll::Pending,
6870        }
6871    }
6872}
6873
6874fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
6875    Error::NonDeterministicReplay(ReplayFailure::new(
6876        "recorded_command_mismatch",
6877        Some(recorded.sequence()),
6878        Some(recorded.shape().to_string()),
6879        Some(actual.into()),
6880        "current workflow command does not match the recorded durable command sequence",
6881    ))
6882}
6883
6884pub struct SignalCall {
6885    ctx: WorkflowContext,
6886    signal_name: String,
6887    opened_wait: bool,
6888    matched_pending: bool,
6889}
6890
6891impl SignalCall {
6892    fn poll_avro_value(
6893        mut self: Pin<&mut Self>,
6894        _cx: &mut TaskContext<'_>,
6895    ) -> Poll<Result<Vec<AvroValue>>> {
6896        if self.matched_pending {
6897            return Poll::Pending;
6898        }
6899
6900        let ctx = self.ctx.clone();
6901        let mut state = match ctx.state.lock() {
6902            Ok(state) => state,
6903            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
6904        };
6905
6906        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
6907            match recorded {
6908                RecordedCommand::SignalWait {
6909                    sequence,
6910                    signal_name,
6911                    value,
6912                } => {
6913                    if signal_name != self.signal_name {
6914                        return Poll::Ready(Err(Error::NonDeterministicReplay(
6915                            ReplayFailure::new(
6916                                "recorded_command_detail_mismatch",
6917                                Some(sequence),
6918                                Some(format!("signal wait:{signal_name}")),
6919                                Some(format!("signal wait:{}", self.signal_name)),
6920                                "recorded signal name differs from the current workflow command",
6921                            ),
6922                        )));
6923                    }
6924
6925                    state.command_cursor += 1;
6926                    if let Some(value) = value {
6927                        return Poll::Ready(Ok(value));
6928                    }
6929                    if state
6930                        .resume_signal
6931                        .as_ref()
6932                        .is_some_and(|signal| signal.signal_name == self.signal_name)
6933                    {
6934                        let signal = state
6935                            .resume_signal
6936                            .take()
6937                            .expect("matching resume signal is present");
6938                        return Poll::Ready(Ok(signal.arguments));
6939                    }
6940
6941                    state.matched_recorded_pending = true;
6942                    self.opened_wait = true;
6943                    self.matched_pending = true;
6944                    return Poll::Pending;
6945                }
6946                other => {
6947                    return Poll::Ready(Err(command_mismatch(
6948                        &other,
6949                        format!("signal wait:{}", self.signal_name),
6950                    )));
6951                }
6952            }
6953        }
6954
6955        if state
6956            .resume_signal
6957            .as_ref()
6958            .is_some_and(|signal| signal.signal_name == self.signal_name)
6959        {
6960            let signal = state
6961                .resume_signal
6962                .take()
6963                .expect("matching resume signal is present");
6964            return Poll::Ready(Ok(signal.arguments));
6965        }
6966
6967        if !self.opened_wait {
6968            state.commands.push(json!({
6969                "type": "open_signal_wait",
6970                "signal_name": self.signal_name
6971            }));
6972            self.opened_wait = true;
6973        }
6974
6975        Poll::Pending
6976    }
6977}
6978
6979impl Future for SignalCall {
6980    type Output = Result<Vec<Value>>;
6981
6982    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
6983        match self.poll_avro_value(cx) {
6984            Poll::Ready(Ok(values)) => Poll::Ready(
6985                values
6986                    .into_iter()
6987                    .map(AvroValue::into_json)
6988                    .collect::<Result<Vec<_>>>(),
6989            ),
6990            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
6991            Poll::Pending => Poll::Pending,
6992        }
6993    }
6994}
6995
6996#[derive(Clone, Debug)]
6997pub struct ActivityContext {
6998    client: Client,
6999    pub task_id: String,
7000    pub activity_attempt_id: String,
7001    pub lease_owner: String,
7002    pub activity_type: String,
7003    pub attempt_number: u64,
7004    pub task_queue: String,
7005    pub worker_id: String,
7006}
7007
7008impl ActivityContext {
7009    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
7010        self.client
7011            .heartbeat_activity_task(
7012                &self.task_id,
7013                &self.activity_attempt_id,
7014                &self.lease_owner,
7015                details,
7016            )
7017            .await
7018    }
7019}
7020
7021fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
7022    match value {
7023        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
7024            value, codec,
7025        )?)),
7026        None => Ok(AvroValue::Array(Vec::new())),
7027    }
7028}
7029
7030fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
7031    let Some(signal_name) = task
7032        .signal_name
7033        .as_deref()
7034        .filter(|value| !value.is_empty())
7035    else {
7036        return Ok(None);
7037    };
7038    let Some(arguments) = task.signal_arguments.as_ref() else {
7039        return Ok(None);
7040    };
7041
7042    let decoded = normalize_avro_arguments(decode_wire_avro_value(arguments, &task.payload_codec)?);
7043    let AvroValue::Array(arguments) = decoded else {
7044        unreachable!("normalize_avro_arguments always returns an array");
7045    };
7046
7047    Ok(Some(ResumeSignal {
7048        signal_name: signal_name.to_string(),
7049        arguments,
7050    }))
7051}
7052
7053fn recorded_commands(
7054    events: &[HistoryEvent],
7055    fallback_codec: &str,
7056    parent: WorkflowIdentity,
7057) -> Result<Vec<RecordedCommand>> {
7058    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
7059    let mut last_new_sequence = None;
7060
7061    for event in events {
7062        let is_activity = matches!(
7063            event.event_type.as_str(),
7064            "ActivityScheduled"
7065                | "ActivityStarted"
7066                | "ActivityHeartbeatRecorded"
7067                | "ActivityRetryScheduled"
7068                | "ActivityCompleted"
7069                | "ActivityFailed"
7070                | "ActivityCancelled"
7071                | "ActivityTimedOut"
7072        );
7073        let is_workflow_timer = matches!(
7074            event.event_type.as_str(),
7075            "TimerScheduled" | "TimerCancelled" | "TimerFired"
7076        ) && !is_internal_timer_event(event);
7077        let is_child_workflow = matches!(
7078            event.event_type.as_str(),
7079            "ChildWorkflowScheduled"
7080                | "ChildRunCompleted"
7081                | "ChildRunFailed"
7082                | "ChildRunCancelled"
7083                | "ChildRunTerminated"
7084        );
7085        let is_signal_wait = is_recorded_signal_wait_event(event);
7086        let is_side_effect = event.event_type == "SideEffectRecorded";
7087        let is_version_marker = event.event_type == "VersionMarkerRecorded";
7088        if !is_activity
7089            && !is_workflow_timer
7090            && !is_child_workflow
7091            && !is_signal_wait
7092            && !is_side_effect
7093            && !is_version_marker
7094        {
7095            continue;
7096        }
7097
7098        let sequence = durable_event_sequence(event).ok_or_else(|| {
7099            Error::NonDeterministicReplay(ReplayFailure::new(
7100                "durable_command_sequence_missing",
7101                None,
7102                Some("positive workflow sequence".to_string()),
7103                Some(event.event_type.clone()),
7104                "durable command history event has no workflow sequence",
7105            ))
7106        })?;
7107        if sequence == 0 {
7108            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7109                "durable_command_sequence_invalid",
7110                Some(sequence),
7111                Some("positive workflow sequence".to_string()),
7112                Some(sequence.to_string()),
7113                "durable command history uses an invalid workflow sequence",
7114            )));
7115        }
7116        if !events_by_sequence.contains_key(&sequence) {
7117            if let Some(previous) = last_new_sequence {
7118                if sequence < previous {
7119                    return Err(invalid_recorded_history(
7120                        "durable_command_sequence_mismatch",
7121                        sequence,
7122                        &format!("workflow sequence greater than {previous}"),
7123                        &sequence.to_string(),
7124                        "durable commands are not strictly ordered by their recorded workflow sequence",
7125                    ));
7126                }
7127            }
7128            last_new_sequence = Some(sequence);
7129        }
7130        events_by_sequence.entry(sequence).or_default().push(event);
7131    }
7132
7133    let commands: Vec<RecordedCommand> = events_by_sequence
7134        .into_iter()
7135        .map(|(sequence, sequence_events)| {
7136            let activity_events: Vec<_> = sequence_events
7137                .iter()
7138                .copied()
7139                .filter(|event| event.event_type.starts_with("Activity"))
7140                .collect();
7141            let timer_events: Vec<_> = sequence_events
7142                .iter()
7143                .copied()
7144                .filter(|event| event.event_type.starts_with("Timer"))
7145                .collect();
7146            let child_events: Vec<_> = sequence_events
7147                .iter()
7148                .copied()
7149                .filter(|event| {
7150                    event.event_type == "ChildWorkflowScheduled"
7151                        || event.event_type.starts_with("ChildRun")
7152                })
7153                .collect();
7154            let signal_wait_events: Vec<_> = sequence_events
7155                .iter()
7156                .copied()
7157                .filter(|event| is_recorded_signal_wait_event(event))
7158                .collect();
7159            let side_effect_events: Vec<_> = sequence_events
7160                .iter()
7161                .copied()
7162                .filter(|event| event.event_type == "SideEffectRecorded")
7163                .collect();
7164            let version_marker_events: Vec<_> = sequence_events
7165                .iter()
7166                .copied()
7167                .filter(|event| event.event_type == "VersionMarkerRecorded")
7168                .collect();
7169
7170            let command_kind_count = usize::from(!activity_events.is_empty())
7171                + usize::from(!timer_events.is_empty())
7172                + usize::from(!child_events.is_empty())
7173                + usize::from(!signal_wait_events.is_empty())
7174                + usize::from(!side_effect_events.is_empty())
7175                + usize::from(!version_marker_events.is_empty());
7176            if command_kind_count > 1 {
7177                let actual = [
7178                    (!activity_events.is_empty()).then_some("activity"),
7179                    (!timer_events.is_empty()).then_some("timer"),
7180                    (!child_events.is_empty()).then_some("child workflow"),
7181                    (!signal_wait_events.is_empty()).then_some("signal wait"),
7182                    (!side_effect_events.is_empty()).then_some("side effect"),
7183                    (!version_marker_events.is_empty()).then_some("version marker"),
7184                ]
7185                .into_iter()
7186                .flatten()
7187                .collect::<Vec<_>>()
7188                .join(" and ");
7189                return Err(invalid_recorded_history(
7190                    "durable_command_sequence_collision",
7191                    sequence,
7192                    "one durable command kind",
7193                    &actual,
7194                    "one workflow sequence records more than one durable command kind",
7195                ));
7196            }
7197
7198            if !activity_events.is_empty() {
7199                let scheduled_count = activity_events
7200                    .iter()
7201                    .filter(|event| event.event_type == "ActivityScheduled")
7202                    .count();
7203                if scheduled_count > 1 {
7204                    return Err(invalid_recorded_history(
7205                        "duplicate_activity_schedule",
7206                        sequence,
7207                        "at most one ActivityScheduled event",
7208                        "multiple ActivityScheduled events",
7209                        "activity history schedules more than one command at one workflow sequence",
7210                    ));
7211                }
7212                let activity_type = activity_events.iter().find_map(|event| {
7213                    event
7214                        .payload
7215                        .get("activity_type")
7216                        .or_else(|| event.payload.get("activity_name"))
7217                        .and_then(Value::as_str)
7218                        .map(str::to_string)
7219                });
7220                if activity_events.iter().filter_map(|event| {
7221                    event
7222                        .payload
7223                        .get("activity_type")
7224                        .or_else(|| event.payload.get("activity_name"))
7225                        .and_then(Value::as_str)
7226                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
7227                    return Err(invalid_recorded_history(
7228                        "activity_identity_mismatch",
7229                        sequence,
7230                        activity_type.as_deref().unwrap_or("one activity identity"),
7231                        "conflicting activity identities",
7232                        "activity lifecycle events at one workflow sequence disagree on identity",
7233                    ));
7234                }
7235                let terminal: Vec<_> = activity_events
7236                    .iter()
7237                    .copied()
7238                    .filter(|event| {
7239                        matches!(
7240                            event.event_type.as_str(),
7241                            "ActivityCompleted"
7242                                | "ActivityFailed"
7243                                | "ActivityCancelled"
7244                                | "ActivityTimedOut"
7245                        )
7246                    })
7247                    .collect();
7248                if terminal.len() > 1 {
7249                    return Err(invalid_recorded_history(
7250                        "duplicate_activity_terminal_event",
7251                        sequence,
7252                        "at most one terminal activity event",
7253                        "multiple terminal activity events",
7254                        "activity history settles one command more than once",
7255                    ));
7256                }
7257                let outcome = terminal
7258                    .first()
7259                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
7260                    .transpose()?;
7261                let options = activity_events
7262                    .iter()
7263                    .find(|event| event.event_type == "ActivityScheduled")
7264                    .and_then(|event| event.payload.get("activity"))
7265                    .and_then(Value::as_object)
7266                    .map(|activity| RecordedActivityOptions {
7267                        task_queue: recorded_optional_string(activity, "queue"),
7268                        execution_mode: recorded_optional_string(activity, "execution_mode"),
7269                        retry_policy: recorded_activity_retry_snapshot(
7270                            activity.get("retry_policy"),
7271                        ),
7272                    });
7273                return Ok(RecordedCommand::Activity {
7274                    sequence,
7275                    activity_type,
7276                    options,
7277                    outcome,
7278                });
7279            }
7280
7281            if !child_events.is_empty() {
7282                let scheduled: Vec<_> = child_events
7283                    .iter()
7284                    .copied()
7285                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
7286                    .collect();
7287                if scheduled.len() != 1 {
7288                    return Err(invalid_recorded_history(
7289                        "child_workflow_schedule_missing_or_duplicate",
7290                        sequence,
7291                        "one ChildWorkflowScheduled event",
7292                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
7293                        "child workflow replay requires exactly one recorded schedule event",
7294                    ));
7295                }
7296                let workflow_type = child_events.iter().find_map(|event| {
7297                    event
7298                        .payload
7299                        .get("child_workflow_type")
7300                        .or_else(|| event.payload.get("workflow_type"))
7301                        .and_then(Value::as_str)
7302                        .filter(|value| !value.is_empty())
7303                        .map(str::to_string)
7304                });
7305                if child_events
7306                    .iter()
7307                    .filter_map(|event| {
7308                        event
7309                            .payload
7310                            .get("child_workflow_type")
7311                            .or_else(|| event.payload.get("workflow_type"))
7312                            .and_then(Value::as_str)
7313                    })
7314                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
7315                {
7316                    return Err(invalid_recorded_history(
7317                        "child_workflow_identity_mismatch",
7318                        sequence,
7319                        workflow_type
7320                            .as_deref()
7321                            .unwrap_or("one child workflow type"),
7322                        "conflicting child workflow types",
7323                        "child workflow lifecycle events at one sequence disagree on type",
7324                    ));
7325                }
7326                let mut outcomes = child_workflow_outcomes(
7327                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
7328                    fallback_codec,
7329                    parent.clone(),
7330                )?;
7331                if outcomes.len() > 1 {
7332                    return Err(invalid_recorded_history(
7333                        "duplicate_child_workflow_terminal_event",
7334                        sequence,
7335                        "at most one terminal child event",
7336                        "multiple terminal child events",
7337                        "child workflow history settles one command more than once",
7338                    ));
7339                }
7340                return Ok(RecordedCommand::ChildWorkflow {
7341                    sequence,
7342                    workflow_type,
7343                    outcome: outcomes.pop(),
7344                });
7345            }
7346
7347            if !signal_wait_events.is_empty() {
7348                let opened: Vec<_> = signal_wait_events
7349                    .iter()
7350                    .copied()
7351                    .filter(|event| event.event_type == "SignalWaitOpened")
7352                    .collect();
7353                if opened.len() != 1 {
7354                    return Err(invalid_recorded_history(
7355                        "signal_wait_open_missing_or_duplicate",
7356                        sequence,
7357                        "one SignalWaitOpened event",
7358                        &format!("{} SignalWaitOpened events", opened.len()),
7359                        "signal replay requires exactly one canonical wait-open event",
7360                    ));
7361                }
7362
7363                let applied: Vec<_> = signal_wait_events
7364                    .iter()
7365                    .copied()
7366                    .filter(|event| event.event_type == "SignalApplied")
7367                    .collect();
7368                if applied.len() > 1 {
7369                    return Err(invalid_recorded_history(
7370                        "duplicate_signal_wait_apply",
7371                        sequence,
7372                        "at most one SignalApplied event",
7373                        "multiple SignalApplied events",
7374                        "signal history applies one durable wait more than once",
7375                    ));
7376                }
7377
7378                let signal_names = signal_wait_events
7379                    .iter()
7380                    .map(|event| required_signal_wait_name(event, sequence))
7381                    .collect::<Result<Vec<_>>>()?;
7382                let signal_name = signal_names
7383                    .first()
7384                    .expect("signal wait events are not empty")
7385                    .clone();
7386                if signal_names.iter().any(|candidate| candidate != &signal_name) {
7387                    return Err(invalid_recorded_history(
7388                        "signal_wait_identity_mismatch",
7389                        sequence,
7390                        &signal_name,
7391                        "conflicting signal names",
7392                        "signal wait lifecycle events at one workflow sequence disagree on identity",
7393                    ));
7394                }
7395                let value = applied
7396                    .first()
7397                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
7398                    .transpose()?;
7399                return Ok(RecordedCommand::SignalWait {
7400                    sequence,
7401                    signal_name,
7402                    value,
7403                });
7404            }
7405
7406            if !side_effect_events.is_empty() {
7407                if side_effect_events.len() != 1 {
7408                    return Err(invalid_recorded_history(
7409                        "duplicate_side_effect_record",
7410                        sequence,
7411                        "one SideEffectRecorded event",
7412                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
7413                        "side-effect history records one workflow command more than once",
7414                    ));
7415                }
7416                let event = side_effect_events[0];
7417                let result = event.payload.get("result").ok_or_else(|| {
7418                    invalid_recorded_history(
7419                        "side_effect_result_missing",
7420                        sequence,
7421                        "recorded result payload",
7422                        "missing result",
7423                        "side-effect history is missing its recorded value",
7424                    )
7425                })?;
7426                let has_published_envelope = result.as_str().is_some()
7427                    || result.as_object().is_some_and(|envelope| {
7428                        envelope.get("codec").and_then(Value::as_str).is_some()
7429                            && envelope.get("blob").and_then(Value::as_str).is_some()
7430                    });
7431                if !has_published_envelope {
7432                    return Err(invalid_recorded_history(
7433                        "side_effect_payload_malformed",
7434                        sequence,
7435                        "payload blob or {codec, blob} envelope",
7436                        &result.to_string(),
7437                        "side-effect history result does not use a published payload envelope",
7438                    ));
7439                }
7440                let codec = event
7441                    .payload
7442                    .get("payload_codec")
7443                    .and_then(Value::as_str)
7444                    .unwrap_or(fallback_codec);
7445                let value = decode_wire_avro_value(result, codec).map_err(|error| {
7446                    invalid_recorded_history(
7447                        "side_effect_payload_incompatible",
7448                        sequence,
7449                        &format!("valid {codec} payload envelope"),
7450                        &error.to_string(),
7451                        "side-effect history payload cannot be decoded with its recorded codec",
7452                    )
7453                })?;
7454                return Ok(RecordedCommand::SideEffect { sequence, value });
7455            }
7456
7457            if !version_marker_events.is_empty() {
7458                if version_marker_events.len() != 1 {
7459                    return Err(invalid_recorded_history(
7460                        "duplicate_version_marker_record",
7461                        sequence,
7462                        "one VersionMarkerRecorded event",
7463                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
7464                        "version-marker history records one workflow command more than once",
7465                    ));
7466                }
7467                let payload = &version_marker_events[0].payload;
7468                let change_id = payload
7469                    .get("change_id")
7470                    .and_then(Value::as_str)
7471                    .filter(|value| !value.is_empty())
7472                    .map(str::to_string)
7473                    .ok_or_else(|| {
7474                        invalid_recorded_history(
7475                            "version_marker_field_missing",
7476                            sequence,
7477                            "non-empty change_id",
7478                            "missing or invalid change_id",
7479                            "version-marker history is missing its stable change ID",
7480                        )
7481                    })?;
7482                let version = required_version_i32(payload, "version", sequence)?;
7483                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
7484                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
7485                if min_supported > max_supported || version < min_supported || version > max_supported {
7486                    return Err(invalid_recorded_history(
7487                        "version_marker_history_range_invalid",
7488                        sequence,
7489                        "min_supported <= version <= max_supported",
7490                        &format!("{min_supported} <= {version} <= {max_supported}"),
7491                        "recorded version marker contains an internally incompatible range",
7492                    ));
7493                }
7494                return Ok(RecordedCommand::VersionMarker {
7495                    sequence,
7496                    change_id,
7497                    version,
7498                });
7499            }
7500
7501            let scheduled: Vec<_> = timer_events
7502                .iter()
7503                .copied()
7504                .filter(|event| event.event_type == "TimerScheduled")
7505                .collect();
7506            let fired: Vec<_> = timer_events
7507                .iter()
7508                .copied()
7509                .filter(|event| event.event_type == "TimerFired")
7510                .collect();
7511            if scheduled.len() != 1 {
7512                return Err(invalid_recorded_history(
7513                    "timer_schedule_missing_or_duplicate",
7514                    sequence,
7515                    "one TimerScheduled event",
7516                    &format!("{} TimerScheduled events", scheduled.len()),
7517                    "timer replay requires exactly one recorded schedule event",
7518                ));
7519            }
7520            if fired.len() > 1 {
7521                return Err(invalid_recorded_history(
7522                    "duplicate_timer_fire",
7523                    sequence,
7524                    "at most one TimerFired event",
7525                    "multiple TimerFired events",
7526                    "timer history contains more than one fire event for a workflow sequence",
7527                ));
7528            }
7529
7530            let scheduled = scheduled[0];
7531            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
7532            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
7533            if let Some(fired) = fired.first() {
7534                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
7535                if fired_timer_id != timer_id {
7536                    return Err(invalid_recorded_history(
7537                        "timer_identity_mismatch",
7538                        sequence,
7539                        &timer_id,
7540                        &fired_timer_id,
7541                        "TimerFired does not correspond to the recorded TimerScheduled event",
7542                    ));
7543                }
7544                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
7545                if fired_delay != delay_seconds {
7546                    return Err(invalid_recorded_history(
7547                        "timer_history_delay_mismatch",
7548                        sequence,
7549                        &delay_seconds.to_string(),
7550                        &fired_delay.to_string(),
7551                        "TimerScheduled and TimerFired record different delays",
7552                    ));
7553                }
7554            }
7555
7556            Ok(RecordedCommand::Timer {
7557                sequence,
7558                delay_seconds,
7559                fired: !fired.is_empty(),
7560            })
7561        })
7562        .collect::<Result<_>>()?;
7563
7564    let mut marker_sequences = HashMap::new();
7565    for command in &commands {
7566        if let RecordedCommand::VersionMarker {
7567            sequence,
7568            change_id,
7569            ..
7570        } = command
7571        {
7572            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
7573                return Err(invalid_recorded_history(
7574                    "duplicate_version_marker",
7575                    *sequence,
7576                    &format!("one marker for change ID {change_id:?}"),
7577                    &format!("markers at sequences {first_sequence} and {sequence}"),
7578                    "workflow history contains duplicate markers for one stable change ID",
7579                ));
7580            }
7581        }
7582    }
7583
7584    Ok(commands)
7585}
7586
7587fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
7588    payload
7589        .get(field)
7590        .and_then(Value::as_i64)
7591        .and_then(|value| i32::try_from(value).ok())
7592        .ok_or_else(|| {
7593            invalid_recorded_history(
7594                "version_marker_field_missing",
7595                sequence,
7596                &format!("integer {field}"),
7597                "missing or out-of-range integer",
7598                "version-marker history is missing a required integer field",
7599            )
7600        })
7601}
7602
7603fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
7604    event
7605        .payload
7606        .get("sequence")
7607        .or_else(|| event.payload.get("workflow_sequence"))
7608        .or_else(|| event.raw.get("sequence"))
7609        .or_else(|| event.raw.get("workflow_sequence"))
7610        .and_then(value_as_u64)
7611}
7612
7613fn is_internal_timer_event(event: &HistoryEvent) -> bool {
7614    matches!(
7615        event
7616            .payload
7617            .get("timer_kind")
7618            .or_else(|| event.raw.get("timer_kind"))
7619            .and_then(Value::as_str),
7620        Some("condition_timeout" | "signal_timeout")
7621    )
7622}
7623
7624fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
7625    event
7626        .payload
7627        .get("signal_name")
7628        .or_else(|| event.raw.get("signal_name"))
7629        .and_then(Value::as_str)
7630        .filter(|value| !value.is_empty())
7631        .map(str::to_string)
7632        .ok_or_else(|| {
7633            invalid_recorded_history(
7634                "signal_wait_name_missing",
7635                sequence,
7636                "non-empty signal_name",
7637                &event.event_type,
7638                "canonical signal-wait history is missing its signal identity",
7639            )
7640        })
7641}
7642
7643fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
7644    matches!(
7645        event.event_type.as_str(),
7646        "SignalWaitOpened" | "SignalApplied"
7647    )
7648}
7649
7650fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
7651    event
7652        .payload
7653        .get(field)
7654        .and_then(Value::as_str)
7655        .filter(|value| !value.is_empty())
7656        .map(str::to_string)
7657        .ok_or_else(|| {
7658            invalid_recorded_history(
7659                "timer_history_field_missing",
7660                sequence,
7661                field,
7662                &event.event_type,
7663                "timer history is missing a required identity field",
7664            )
7665        })
7666}
7667
7668fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
7669    event
7670        .payload
7671        .get(field)
7672        .and_then(value_as_u64)
7673        .ok_or_else(|| {
7674            invalid_recorded_history(
7675                "timer_history_field_missing",
7676                sequence,
7677                field,
7678                &event.event_type,
7679                "timer history is missing a required numeric field",
7680            )
7681        })
7682}
7683
7684fn invalid_recorded_history(
7685    reason: &str,
7686    sequence: u64,
7687    expected: &str,
7688    actual: &str,
7689    message: &str,
7690) -> Error {
7691    Error::NonDeterministicReplay(ReplayFailure::new(
7692        reason,
7693        Some(sequence),
7694        Some(expected.to_string()),
7695        Some(actual.to_string()),
7696        message,
7697    ))
7698}
7699
7700type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
7701
7702fn activity_outcome(
7703    event: &HistoryEvent,
7704    fallback_codec: &str,
7705    recorded_activity_type: Option<String>,
7706) -> Result<ActivityOutcome> {
7707    if event.event_type == "ActivityCompleted" {
7708        let codec = event
7709            .payload
7710            .get("payload_codec")
7711            .and_then(Value::as_str)
7712            .unwrap_or(fallback_codec);
7713        return Ok(Ok(decode_wire_avro_value(
7714            event.payload.get("result").unwrap_or(&Value::Null),
7715            codec,
7716        )?));
7717    }
7718
7719    let payload = &event.payload;
7720    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
7721        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
7722        "ActivityCancelled" => (
7723            ActivityFailureKind::Cancelled,
7724            "cancelled",
7725            "activity was cancelled",
7726        ),
7727        "ActivityTimedOut" => (
7728            ActivityFailureKind::TimedOut,
7729            "timeout",
7730            "activity timed out",
7731        ),
7732        _ => unreachable!("activity_outcome is called only for terminal activity events"),
7733    };
7734    let exception = payload
7735        .get("exception")
7736        .filter(|value| !value.is_null())
7737        .cloned();
7738    let failure_category = payload_string(payload, "failure_category");
7739    let timeout_kind = payload_string(payload, "timeout_kind");
7740    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
7741        ActivityFailureKind::Failed => failure_category
7742            .clone()
7743            .unwrap_or_else(|| fallback_reason.to_string()),
7744        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
7745        ActivityFailureKind::TimedOut => timeout_kind
7746            .clone()
7747            .unwrap_or_else(|| fallback_reason.to_string()),
7748    });
7749    let message = payload_string(payload, "message")
7750        .or_else(|| {
7751            exception
7752                .as_ref()
7753                .and_then(|value| payload_string(value, "message"))
7754        })
7755        .unwrap_or_else(|| fallback_message.to_string());
7756
7757    Ok(Err(ActivityFailure {
7758        kind,
7759        reason,
7760        message,
7761        activity_execution_id: payload_string(payload, "activity_execution_id"),
7762        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
7763        activity_type: payload_string(payload, "activity_type")
7764            .or_else(|| payload_string(payload, "activity_name"))
7765            .or(recorded_activity_type),
7766        activity_class: payload_string(payload, "activity_class"),
7767        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
7768        failure_id: payload_string(payload, "failure_id"),
7769        failure_category,
7770        timeout_kind,
7771        non_retryable: payload
7772            .get("non_retryable")
7773            .and_then(Value::as_bool)
7774            .unwrap_or(false),
7775        exception_type: payload_string(payload, "exception_type").or_else(|| {
7776            exception
7777                .as_ref()
7778                .and_then(|value| payload_string(value, "type"))
7779        }),
7780        exception_class: payload_string(payload, "exception_class").or_else(|| {
7781            exception
7782                .as_ref()
7783                .and_then(|value| payload_string(value, "class"))
7784        }),
7785        code: payload
7786            .get("code")
7787            .filter(|value| !value.is_null())
7788            .cloned(),
7789        exception,
7790    }))
7791}
7792
7793type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
7794
7795fn child_workflow_outcomes(
7796    events: &[HistoryEvent],
7797    fallback_codec: &str,
7798    parent: WorkflowIdentity,
7799) -> Result<Vec<ChildWorkflowOutcome>> {
7800    let mut outcomes = Vec::new();
7801
7802    for event in events {
7803        let kind = match event.event_type.as_str() {
7804            "ChildRunCompleted" => None,
7805            "ChildRunFailed" => Some((
7806                ChildWorkflowFailureKind::Failed,
7807                "child_workflow",
7808                "child workflow failed",
7809            )),
7810            "ChildRunCancelled" => Some((
7811                ChildWorkflowFailureKind::Cancelled,
7812                "cancelled",
7813                "child workflow was cancelled",
7814            )),
7815            "ChildRunTerminated" => Some((
7816                ChildWorkflowFailureKind::Terminated,
7817                "terminated",
7818                "child workflow was terminated",
7819            )),
7820            _ => continue,
7821        };
7822        let payload = &event.payload;
7823        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
7824        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
7825        let child_workflow_type = payload_string(payload, "child_workflow_type");
7826
7827        if let Some((kind, reason, fallback_message)) = kind {
7828            let exception = payload
7829                .get("exception")
7830                .filter(|value| !value.is_null())
7831                .cloned();
7832            let message = payload_string(payload, "message")
7833                .or_else(|| {
7834                    exception
7835                        .as_ref()
7836                        .and_then(|value| payload_string(value, "message"))
7837                })
7838                .unwrap_or_else(|| fallback_message.to_string());
7839            let exception_type = payload_string(payload, "exception_type").or_else(|| {
7840                exception
7841                    .as_ref()
7842                    .and_then(|value| payload_string(value, "type"))
7843            });
7844            let exception_class = payload_string(payload, "exception_class").or_else(|| {
7845                exception
7846                    .as_ref()
7847                    .and_then(|value| payload_string(value, "class"))
7848            });
7849            outcomes.push(Err(ChildWorkflowFailure {
7850                kind,
7851                reason: reason.to_string(),
7852                message,
7853                parent_workflow_id: parent.workflow_id.clone(),
7854                parent_workflow_run_id: parent.run_id.clone(),
7855                child_workflow_id,
7856                child_workflow_run_id,
7857                child_workflow_type,
7858                failure_id: payload_string(payload, "failure_id"),
7859                failure_category: payload_string(payload, "failure_category"),
7860                exception_type,
7861                exception_class,
7862                non_retryable: payload
7863                    .get("non_retryable")
7864                    .and_then(Value::as_bool)
7865                    .unwrap_or(false),
7866                code: payload
7867                    .get("code")
7868                    .filter(|value| !value.is_null())
7869                    .cloned(),
7870                exception,
7871            }));
7872            continue;
7873        }
7874
7875        let codec = payload
7876            .get("payload_codec")
7877            .and_then(Value::as_str)
7878            .unwrap_or(fallback_codec);
7879        let result = payload
7880            .get("result")
7881            .or_else(|| payload.get("output"))
7882            .unwrap_or(&Value::Null);
7883        outcomes.push(Ok(ChildWorkflowAvroResult {
7884            parent: parent.clone(),
7885            child: WorkflowIdentity {
7886                workflow_id: child_workflow_id,
7887                run_id: child_workflow_run_id,
7888            },
7889            child_workflow_type,
7890            result: decode_wire_avro_value(result, codec)?,
7891        }));
7892    }
7893
7894    Ok(outcomes)
7895}
7896
7897fn payload_string(payload: &Value, key: &str) -> Option<String> {
7898    payload
7899        .get(key)
7900        .and_then(Value::as_str)
7901        .filter(|value| !value.is_empty())
7902        .map(str::to_string)
7903}
7904
7905fn workflow_failure_command(error: &Error) -> Value {
7906    let (exception_type, exception_class, properties) = match error {
7907        Error::ActivityFailed(failure) => (
7908            match failure.kind {
7909                ActivityFailureKind::Failed => "ActivityFailed",
7910                ActivityFailureKind::Cancelled => "ActivityCancelled",
7911                ActivityFailureKind::TimedOut => "ActivityTimedOut",
7912            },
7913            "durable_workflow::ActivityFailure",
7914            json!({
7915                "reason": failure.reason,
7916                "activity_execution_id": failure.activity_execution_id,
7917                "activity_attempt_id": failure.activity_attempt_id,
7918                "activity_type": failure.activity_type,
7919                "activity_class": failure.activity_class,
7920                "attempt_number": failure.attempt_number,
7921                "failure_id": failure.failure_id,
7922                "failure_category": failure.failure_category,
7923                "timeout_kind": failure.timeout_kind,
7924                "activity_non_retryable": failure.non_retryable,
7925                "activity_exception_type": failure.exception_type,
7926                "activity_exception_class": failure.exception_class,
7927                "activity_code": failure.code,
7928                "activity_exception": failure.exception,
7929            }),
7930        ),
7931        Error::ChildWorkflowFailed(failure) => (
7932            match failure.kind {
7933                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
7934                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
7935                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
7936            },
7937            "durable_workflow::ChildWorkflowFailure",
7938            json!({
7939                "reason": failure.reason,
7940                "parent_workflow_id": failure.parent_workflow_id,
7941                "parent_workflow_run_id": failure.parent_workflow_run_id,
7942                "child_workflow_id": failure.child_workflow_id,
7943                "child_workflow_run_id": failure.child_workflow_run_id,
7944                "child_workflow_type": failure.child_workflow_type,
7945                "failure_id": failure.failure_id,
7946                "failure_category": failure.failure_category,
7947                "child_exception_type": failure.exception_type,
7948                "child_exception_class": failure.exception_class,
7949                "child_non_retryable": failure.non_retryable,
7950                "child_code": failure.code,
7951                "child_exception": failure.exception,
7952            }),
7953        ),
7954        Error::NonDeterministicReplay(_) => (
7955            "NonDeterministicReplay",
7956            "durable_workflow::Error",
7957            Value::Null,
7958        ),
7959        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
7960    };
7961    let non_retryable = match error {
7962        Error::ActivityFailed(failure) => failure.non_retryable,
7963        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
7964        Error::NonDeterministicReplay(_) => true,
7965        _ => false,
7966    };
7967
7968    json!({
7969        "type": "fail_workflow",
7970        "message": error.to_string(),
7971        "exception_type": exception_type,
7972        "exception_class": exception_class,
7973        "non_retryable": non_retryable,
7974        "exception": {
7975            "type": exception_type,
7976            "class": exception_class,
7977            "message": error.to_string(),
7978            "properties": properties,
7979        }
7980    })
7981}
7982
7983fn workflow_task_integrity_error(error: &Error) -> bool {
7984    matches!(
7985        error,
7986        Error::NonDeterministicReplay(_) | Error::Protocol(_) | Error::WorkflowStatePoisoned
7987    )
7988}
7989
7990fn decode_signal_event_arguments(
7991    event: &HistoryEvent,
7992    fallback_codec: &str,
7993) -> Result<Vec<AvroValue>> {
7994    let codec = event
7995        .payload
7996        .get("payload_codec")
7997        .and_then(Value::as_str)
7998        .unwrap_or(fallback_codec);
7999    let raw = event
8000        .payload
8001        .get("value")
8002        .or_else(|| event.payload.get("input"))
8003        .or_else(|| event.payload.get("arguments"));
8004    let decoded = match raw.filter(|value| !value.is_null()) {
8005        Some(value) => decode_wire_avro_value(value, codec)?,
8006        None => AvroValue::Array(Vec::new()),
8007    };
8008    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
8009        unreachable!("normalize_avro_arguments always returns an array");
8010    };
8011    Ok(arguments)
8012}
8013
8014fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8015    let Some(export_events) = task
8016        .history_export
8017        .as_ref()
8018        .and_then(|export| export.get("history_events"))
8019        .and_then(Value::as_array)
8020    else {
8021        return Ok(());
8022    };
8023
8024    if export_events.len() > task.history_events.len() {
8025        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
8026    }
8027
8028    Ok(())
8029}
8030
8031fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
8032    let Some(export) = task.history_export.as_ref() else {
8033        return Ok(());
8034    };
8035    let signals = export
8036        .get("signals")
8037        .and_then(Value::as_array)
8038        .cloned()
8039        .unwrap_or_default();
8040    let activities = export
8041        .get("activities")
8042        .and_then(Value::as_array)
8043        .cloned()
8044        .unwrap_or_default();
8045    let export_codec = export
8046        .get("payloads")
8047        .and_then(|payloads| payloads.get("codec"))
8048        .and_then(Value::as_str)
8049        .unwrap_or(&task.payload_codec)
8050        .to_string();
8051    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
8052
8053    for event in &mut task.history_events {
8054        if event.event_type == "ActivityCompleted" {
8055            let sequence = event
8056                .payload
8057                .get("sequence")
8058                .or_else(|| event.payload.get("workflow_sequence"))
8059                .and_then(value_as_u64);
8060            let Some(activity) = sequence.and_then(|sequence| {
8061                activities.iter().find(|activity| {
8062                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
8063                })
8064            }) else {
8065                continue;
8066            };
8067            let Some(payload) = event.payload.as_object_mut() else {
8068                continue;
8069            };
8070            if missing_payload(payload.get("result")) {
8071                if let Some(result) = activity
8072                    .get("result")
8073                    .filter(|value| !missing_payload(Some(value)))
8074                {
8075                    payload.insert("result".to_string(), result.clone());
8076                }
8077            }
8078            for field in ["payload_codec", "activity_type"] {
8079                if payload
8080                    .get(field)
8081                    .and_then(Value::as_str)
8082                    .unwrap_or_default()
8083                    .is_empty()
8084                {
8085                    if let Some(value) = activity.get(field) {
8086                        payload.insert(field.to_string(), value.clone());
8087                    }
8088                }
8089            }
8090            continue;
8091        }
8092
8093        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
8094            continue;
8095        }
8096        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8097        let command_id = event
8098            .payload
8099            .get("workflow_command_id")
8100            .or_else(|| event.raw.get("workflow_command_id"))
8101            .and_then(Value::as_str);
8102        let signal_name = event
8103            .payload
8104            .get("signal_name")
8105            .and_then(Value::as_str)
8106            .unwrap_or_default()
8107            .to_string();
8108        let matched = signals
8109            .iter()
8110            .find(|signal| {
8111                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
8112            })
8113            .or_else(|| {
8114                signals.iter().find(|signal| {
8115                    command_id.is_some()
8116                        && signal.get("command_id").and_then(Value::as_str) == command_id
8117                })
8118            })
8119            .or_else(|| {
8120                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
8121                let signal = signals
8122                    .iter()
8123                    .filter(|signal| {
8124                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
8125                    })
8126                    .nth(*offset);
8127                if signal.is_some() {
8128                    *offset += 1;
8129                }
8130                signal
8131            });
8132        let Some(signal) = matched else {
8133            continue;
8134        };
8135        let signal_codec = signal
8136            .get("payload_codec")
8137            .and_then(Value::as_str)
8138            .unwrap_or(&export_codec);
8139        let Some(payload) = event.payload.as_object_mut() else {
8140            continue;
8141        };
8142        if missing_payload(payload.get("arguments")) {
8143            if let Some(arguments) = signal
8144                .get("arguments")
8145                .filter(|value| !missing_payload(Some(value)))
8146            {
8147                let envelope = match arguments {
8148                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
8149                    other => other.clone(),
8150                };
8151                payload.insert("arguments".to_string(), envelope);
8152            }
8153        }
8154        if payload
8155            .get("payload_codec")
8156            .and_then(Value::as_str)
8157            .unwrap_or_default()
8158            .is_empty()
8159        {
8160            payload.insert("payload_codec".to_string(), json!(signal_codec));
8161        }
8162    }
8163
8164    Ok(())
8165}
8166
8167fn missing_payload(value: Option<&Value>) -> bool {
8168    match value {
8169        None | Some(Value::Null) => true,
8170        Some(Value::String(value)) => value.is_empty(),
8171        Some(_) => false,
8172    }
8173}
8174
8175fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
8176    let export_signals = task
8177        .history_export
8178        .as_ref()
8179        .and_then(|export| export.get("signals"))
8180        .and_then(Value::as_array)
8181        .cloned()
8182        .unwrap_or_default();
8183    let export_codec = task
8184        .history_export
8185        .as_ref()
8186        .and_then(|export| export.get("payloads"))
8187        .and_then(|payloads| payloads.get("codec"))
8188        .and_then(Value::as_str)
8189        .unwrap_or(&task.payload_codec);
8190    let mut name_offsets: HashMap<String, usize> = HashMap::new();
8191    let mut signals = Vec::new();
8192
8193    for event in &task.history_events {
8194        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
8195            continue;
8196        }
8197
8198        let name = event
8199            .payload
8200            .get("signal_name")
8201            .and_then(Value::as_str)
8202            .unwrap_or_default();
8203        if name.is_empty() {
8204            continue;
8205        }
8206        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
8207        let command_id = event
8208            .payload
8209            .get("workflow_command_id")
8210            .or_else(|| event.raw.get("workflow_command_id"))
8211            .and_then(Value::as_str);
8212        let matched_export = export_signals
8213            .iter()
8214            .find(|candidate| {
8215                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
8216            })
8217            .or_else(|| {
8218                export_signals.iter().find(|candidate| {
8219                    command_id.is_some()
8220                        && candidate.get("command_id").and_then(Value::as_str) == command_id
8221                })
8222            })
8223            .or_else(|| {
8224                let offset = name_offsets.entry(name.to_string()).or_default();
8225                let candidate = export_signals
8226                    .iter()
8227                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
8228                    .nth(*offset);
8229                if candidate.is_some() {
8230                    *offset += 1;
8231                }
8232                candidate
8233            });
8234        let codec = event
8235            .payload
8236            .get("payload_codec")
8237            .and_then(Value::as_str)
8238            .or_else(|| {
8239                matched_export
8240                    .and_then(|signal| signal.get("payload_codec"))
8241                    .and_then(Value::as_str)
8242            })
8243            .unwrap_or(export_codec);
8244        let raw_arguments = event
8245            .payload
8246            .get("value")
8247            .or_else(|| event.payload.get("input"))
8248            .or_else(|| event.payload.get("arguments"))
8249            .filter(|value| !value.is_null())
8250            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
8251        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
8252        let workflow_sequence = event
8253            .payload
8254            .get("workflow_sequence")
8255            .and_then(value_as_u64)
8256            .or_else(|| {
8257                matched_export
8258                    .and_then(|signal| signal.get("workflow_sequence"))
8259                    .and_then(value_as_u64)
8260            });
8261
8262        signals.push(QuerySignal {
8263            id: signal_id.map(str::to_string).or_else(|| {
8264                matched_export
8265                    .and_then(|signal| signal.get("id"))
8266                    .and_then(Value::as_str)
8267                    .map(str::to_string)
8268            }),
8269            name: name.to_string(),
8270            arguments,
8271            avro_arguments,
8272            workflow_sequence,
8273        });
8274    }
8275
8276    if signals.is_empty() {
8277        for signal in export_signals {
8278            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
8279                continue;
8280            }
8281            let Some(name) = signal.get("name").and_then(Value::as_str) else {
8282                continue;
8283            };
8284            let codec = signal
8285                .get("payload_codec")
8286                .and_then(Value::as_str)
8287                .unwrap_or(export_codec);
8288            let (arguments, avro_arguments) =
8289                decode_query_signal_arguments(signal.get("arguments"), codec)?;
8290            signals.push(QuerySignal {
8291                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
8292                name: name.to_string(),
8293                arguments,
8294                avro_arguments,
8295                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
8296            });
8297        }
8298        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
8299    }
8300
8301    Ok(signals)
8302}
8303
8304fn decode_query_signal_arguments(
8305    raw: Option<&Value>,
8306    codec: &str,
8307) -> Result<(Vec<Value>, Vec<AvroValue>)> {
8308    let decoded = match raw.filter(|value| !value.is_null()) {
8309        Some(value) => decode_wire_avro_value(value, codec)?,
8310        None => AvroValue::Array(Vec::new()),
8311    };
8312    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
8313        unreachable!("normalize_avro_arguments always returns an array");
8314    };
8315    let arguments = avro_arguments
8316        .iter()
8317        .cloned()
8318        .map(AvroValue::into_json)
8319        .collect::<Result<Vec<_>>>()?;
8320    Ok((arguments, avro_arguments))
8321}
8322
8323fn value_as_u64(value: &Value) -> Option<u64> {
8324    value
8325        .as_u64()
8326        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
8327}
8328
8329#[cfg(test)]
8330mod tests {
8331    use super::*;
8332    use std::{
8333        io::{Read, Write},
8334        net::{SocketAddr, TcpListener, TcpStream},
8335        sync::atomic::AtomicUsize,
8336        thread,
8337    };
8338
8339    #[test]
8340    fn client_builder_rejects_the_sdk_owned_api_suffix() {
8341        for base_url in [
8342            "http://127.0.0.1:8080/api",
8343            "http://localhost:8080/api/",
8344            "https://runtime.example.test/namespaces/orders/api",
8345        ] {
8346            let error = Client::builder(base_url)
8347                .build()
8348                .expect_err("SDK-owned /api suffix must be rejected during build");
8349
8350            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
8351            assert!(
8352                error.to_string().contains("SDK appends /api automatically"),
8353                "the validation error must explain how to fix the endpoint"
8354            );
8355        }
8356    }
8357
8358    #[test]
8359    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
8360        for (base_url, expected) in [
8361            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
8362            (
8363                "http://localhost:8080/durable-workflow/",
8364                "http://localhost:8080/durable-workflow",
8365            ),
8366            (
8367                "https://runtime.example.test/namespaces/orders",
8368                "https://runtime.example.test/namespaces/orders",
8369            ),
8370            (
8371                "https://runtime.example.test/gateway/api/namespaces/orders",
8372                "https://runtime.example.test/gateway/api/namespaces/orders",
8373            ),
8374            (
8375                "https://api.example.test/runtime/orders/",
8376                "https://api.example.test/runtime/orders",
8377            ),
8378        ] {
8379            let client = Client::builder(base_url)
8380                .build()
8381                .expect("Server and Cloud runtime base URL must remain valid");
8382
8383            assert_eq!(client.base_url, expected);
8384        }
8385    }
8386
8387    fn typed_fidelity_probe() -> AvroValue {
8388        AvroValue::Map(BTreeMap::from([
8389            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
8390            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
8391            (
8392                "numeric".to_string(),
8393                AvroValue::Map(BTreeMap::from([
8394                    ("0".to_string(), AvroValue::String("zero".to_string())),
8395                    ("1".to_string(), AvroValue::String("one".to_string())),
8396                ])),
8397            ),
8398            (
8399                "nested".to_string(),
8400                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
8401                    "enabled".to_string(),
8402                    AvroValue::Boolean(true),
8403                )]))]),
8404            ),
8405            (
8406                "projection_collisions".to_string(),
8407                AvroValue::Array(projection_collision_probe()),
8408            ),
8409        ]))
8410    }
8411
8412    fn projection_collision_probe() -> Vec<AvroValue> {
8413        vec![
8414            AvroValue::Map(BTreeMap::from([
8415                ("$type".to_string(), AvroValue::String("bytes".to_string())),
8416                (
8417                    "base64".to_string(),
8418                    AvroValue::String("ordinary user text".to_string()),
8419                ),
8420            ])),
8421            AvroValue::Map(BTreeMap::from([
8422                ("$type".to_string(), AvroValue::String("map".to_string())),
8423                (
8424                    "entries".to_string(),
8425                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
8426                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
8427                        (
8428                            "value".to_string(),
8429                            AvroValue::String("user map".to_string()),
8430                        ),
8431                    ]))]),
8432                ),
8433            ])),
8434        ]
8435    }
8436
8437    #[derive(Clone, Debug, Default, PartialEq)]
8438    struct ReplayCounterState {
8439        loaded: Option<String>,
8440        count: i64,
8441        finished: bool,
8442    }
8443
8444    fn replay_counter_worker() -> Worker {
8445        let client = Client::new("http://127.0.0.1:8080").expect("client");
8446        let mut worker = Worker::new(client, "rust-workers");
8447        worker.register_replayed_workflow(
8448            "replay-counter",
8449            ReplayCounterState::default,
8450            |ctx, _input, state| async move {
8451                let loaded = ctx.activity("load-counter", json!([])).await?;
8452                state.update(|current| {
8453                    current.loaded = loaded.as_str().map(str::to_string);
8454                })?;
8455                for _ in 0..2 {
8456                    let signal = ctx.wait_signal("increment").await?;
8457                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
8458                    state.update(|current| current.count += amount)?;
8459                }
8460                state.update(|current| current.finished = true)?;
8461                state.read(|current| Ok(json!(current.count)))?
8462            },
8463        );
8464        worker.register_replayed_query::<ReplayCounterState, _, _>(
8465            "replay-counter",
8466            "current",
8467            |_ctx, state, _args| async move {
8468                Ok(json!({
8469                    "loaded": state.loaded,
8470                    "count": state.count,
8471                    "finished": state.finished,
8472                }))
8473            },
8474        );
8475        worker.register_replayed_query::<ReplayCounterState, _, _>(
8476            "replay-counter",
8477            "detached-mutation",
8478            |_ctx, state, _args| async move {
8479                let mut detached = (*state).clone();
8480                detached.count = 999;
8481                Ok(json!(detached.count))
8482            },
8483        );
8484        worker.register_replayed_query::<ReplayCounterState, _, _>(
8485            "replay-counter",
8486            "failed-mutation",
8487            |_ctx, state, _args| async move {
8488                let mut detached = (*state).clone();
8489                detached.count = 999;
8490                Err(Error::WorkerLoop("query refused".to_string()))
8491            },
8492        );
8493        worker
8494    }
8495
8496    fn replay_counter_query(
8497        query_name: &str,
8498        history_events: Value,
8499        run_status: &str,
8500    ) -> QueryTask {
8501        serde_json::from_value(json!({
8502            "query_task_id": format!("query-{query_name}"),
8503            "workflow_type": "replay-counter",
8504            "query_name": query_name,
8505            "payload_codec": "json",
8506            "workflow_arguments": {"codec": "json", "blob": "[]"},
8507            "query_arguments": {"codec": "json", "blob": "[]"},
8508            "history_events": history_events,
8509            "run_status": run_status,
8510        }))
8511        .expect("query task")
8512    }
8513
8514    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
8515        workflow_context_with_codec(history, JSON_CODEC)
8516    }
8517
8518    fn workflow_context_with_codec(
8519        history: Vec<HistoryEvent>,
8520        payload_codec: &str,
8521    ) -> WorkflowContext {
8522        WorkflowContext {
8523            state: Arc::new(Mutex::new(
8524                WorkflowState::new_with_identity(
8525                    history,
8526                    None,
8527                    None,
8528                    "rust-workers".to_string(),
8529                    payload_codec.to_string(),
8530                    None,
8531                )
8532                .expect("valid workflow history"),
8533            )),
8534        }
8535    }
8536
8537    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
8538        HistoryEvent {
8539            event_type: event_type.to_string(),
8540            payload,
8541            raw: HashMap::new(),
8542        }
8543    }
8544
8545    fn workflow_task(
8546        workflow_type: &str,
8547        history_events: Vec<HistoryEvent>,
8548        payload_codec: &str,
8549    ) -> WorkflowTask {
8550        WorkflowTask {
8551            task_id: format!("wft-{workflow_type}"),
8552            workflow_id: Some(format!("wf-{workflow_type}")),
8553            run_id: Some(format!("run-{workflow_type}")),
8554            workflow_type: workflow_type.to_string(),
8555            payload_codec: payload_codec.to_string(),
8556            arguments: Some(
8557                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
8558            ),
8559            total_history_events: Some(history_events.len() as u64),
8560            history_size_bytes: None,
8561            continue_as_new_recommended: None,
8562            history_budget_pressure: None,
8563            history_events,
8564            next_history_page_token: None,
8565            workflow_task_attempt: 1,
8566            workflow_signal_id: None,
8567            signal_name: None,
8568            signal_arguments: None,
8569            workflow_update_id: None,
8570            update_name: None,
8571            lease_owner: Some("rust-worker".to_string()),
8572        }
8573    }
8574
8575    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
8576    struct SideEffectProbe {
8577        request_id: String,
8578        attempt: u32,
8579    }
8580
8581    #[test]
8582    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
8583        let calls = AtomicUsize::new(0);
8584        let ctx = workflow_context(Vec::new());
8585        let value = ctx
8586            .side_effect(|| {
8587                calls.fetch_add(1, Ordering::SeqCst);
8588                SideEffectProbe {
8589                    request_id: "request-42".to_string(),
8590                    attempt: 3,
8591                }
8592            })
8593            .expect("first side effect");
8594        assert_eq!(value.attempt, 3);
8595        assert_eq!(calls.load(Ordering::SeqCst), 1);
8596        let commands = ctx.take_commands().expect("commands");
8597        assert_eq!(commands.len(), 1);
8598        assert_eq!(commands[0]["type"], "record_side_effect");
8599        assert_eq!(
8600            decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("JSON result"),
8601            serde_json::to_value(&value).expect("value")
8602        );
8603
8604        let replay = workflow_context(vec![history_event(
8605            "SideEffectRecorded",
8606            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8607        )]);
8608        let replayed: SideEffectProbe = replay
8609            .side_effect(|| {
8610                calls.fetch_add(1, Ordering::SeqCst);
8611                panic!("committed side-effect callbacks must not run during replay")
8612            })
8613            .expect("replayed side effect");
8614        assert_eq!(replayed, value);
8615        assert_eq!(calls.load(Ordering::SeqCst), 1);
8616        assert!(replay.take_commands().expect("commands").is_empty());
8617        replay.ensure_history_consumed().expect("history consumed");
8618    }
8619
8620    #[test]
8621    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
8622        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8623        let value = ctx
8624            .side_effect(|| SideEffectProbe {
8625                request_id: "avro-request".to_string(),
8626                attempt: 1,
8627            })
8628            .expect("Avro side effect");
8629        let uuid = ctx.uuid_v4().expect("deterministic UUID");
8630        let commands = ctx.take_commands().expect("commands");
8631        assert_eq!(commands.len(), 2);
8632        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
8633        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
8634        assert_eq!(
8635            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
8636            serde_json::to_value(&value).expect("value")
8637        );
8638
8639        let replay = workflow_context_with_codec(
8640            vec![
8641                history_event(
8642                    "SideEffectRecorded",
8643                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8644                ),
8645                history_event(
8646                    "SideEffectRecorded",
8647                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
8648                ),
8649            ],
8650            DEFAULT_CODEC,
8651        );
8652        let replayed: SideEffectProbe = replay
8653            .side_effect(|| panic!("Avro callback must not run"))
8654            .expect("replayed Avro value");
8655        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
8656        assert_eq!(replayed, value);
8657        assert_eq!(replayed_uuid, uuid);
8658        assert!(replay.take_commands().expect("commands").is_empty());
8659    }
8660
8661    #[test]
8662    fn typed_side_effect_replay_preserves_bytes_and_maps() {
8663        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
8664        let value = ctx
8665            .side_effect_avro_value(typed_fidelity_probe)
8666            .expect("typed side effect");
8667        let commands = ctx.take_commands().expect("side-effect command");
8668        assert_eq!(
8669            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
8670                .expect("recorded side effect"),
8671            value
8672        );
8673
8674        let replay = workflow_context_with_codec(
8675            vec![history_event(
8676                "SideEffectRecorded",
8677                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
8678            )],
8679            DEFAULT_CODEC,
8680        );
8681        assert_eq!(
8682            replay
8683                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
8684                .expect("replayed typed side effect"),
8685            value
8686        );
8687    }
8688
8689    #[test]
8690    fn ordered_side_effects_share_the_durable_command_stream() {
8691        let first = encode_value_envelope(&json!("first"), JSON_CODEC).expect("first");
8692        let second = encode_value_envelope(&json!(29), JSON_CODEC).expect("second");
8693        let ctx = workflow_context(vec![
8694            history_event(
8695                "SideEffectRecorded",
8696                json!({"sequence": 1, "result": first}),
8697            ),
8698            history_event(
8699                "SideEffectRecorded",
8700                json!({"sequence": 2, "result": second}),
8701            ),
8702        ]);
8703        let first: String = ctx
8704            .side_effect(|| panic!("first callback must not run"))
8705            .expect("first replay");
8706        let second: i32 = ctx
8707            .side_effect(|| panic!("second callback must not run"))
8708            .expect("second replay");
8709        assert_eq!(first, "first");
8710        assert_eq!(second, 29);
8711        ctx.ensure_history_consumed().expect("ordered history");
8712
8713        let reordered = workflow_context(vec![history_event(
8714            "VersionMarkerRecorded",
8715            json!({
8716                "sequence": 1,
8717                "change_id": "before-side-effect",
8718                "version": 1,
8719                "min_supported": 1,
8720                "max_supported": 1,
8721            }),
8722        )]);
8723        let error = reordered
8724            .side_effect(|| "new".to_string())
8725            .expect_err("command reordering must fail");
8726        assert!(matches!(
8727            error,
8728            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8729                if reason == "recorded_command_mismatch"
8730        ));
8731    }
8732
8733    #[test]
8734    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
8735        let ctx = workflow_context(Vec::new());
8736        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
8737        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
8738        assert!(ctx.patched("new-search").expect("patch"));
8739        ctx.deprecate_patch("new-search").expect("deprecate patch");
8740        let commands = ctx.take_commands().expect("commands");
8741        assert_eq!(commands.len(), 2);
8742        assert_eq!(commands[0]["type"], "record_version_marker");
8743        assert_eq!(commands[0]["version"], 2);
8744        assert_eq!(commands[1]["change_id"], "new-search");
8745
8746        let replay = workflow_context(vec![history_event(
8747            "VersionMarkerRecorded",
8748            json!({
8749                "sequence": 1,
8750                "change_id": "checkout-v2",
8751                "version": 2,
8752                "min_supported": 1,
8753                "max_supported": 2,
8754            }),
8755        )]);
8756        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
8757        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
8758        assert!(replay.take_commands().expect("commands").is_empty());
8759        replay.ensure_history_consumed().expect("history consumed");
8760    }
8761
8762    #[test]
8763    fn version_markers_reject_incompatible_or_malformed_history() {
8764        let incompatible = workflow_context(vec![history_event(
8765            "VersionMarkerRecorded",
8766            json!({
8767                "sequence": 1,
8768                "change_id": "checkout-v2",
8769                "version": 1,
8770                "min_supported": 1,
8771                "max_supported": 2,
8772            }),
8773        )]);
8774        let error = incompatible
8775            .get_version("checkout-v2", 2, 3)
8776            .expect_err("old version is unsupported");
8777        assert!(matches!(
8778            error,
8779            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8780                if reason == "version_marker_incompatible_range"
8781        ));
8782
8783        for (history, reason) in [
8784            (
8785                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
8786                "side_effect_result_missing",
8787            ),
8788            (
8789                vec![history_event(
8790                    "SideEffectRecorded",
8791                    json!({
8792                        "sequence": 1,
8793                        "result": {"codec": "avro", "blob": "not-base64"},
8794                    }),
8795                )],
8796                "side_effect_payload_incompatible",
8797            ),
8798            (
8799                vec![history_event(
8800                    "SideEffectRecorded",
8801                    json!({"sequence": 1, "result": {"unwrapped": true}}),
8802                )],
8803                "side_effect_payload_malformed",
8804            ),
8805            (
8806                vec![history_event(
8807                    "VersionMarkerRecorded",
8808                    json!({
8809                        "sequence": 1,
8810                        "change_id": "change",
8811                        "version": 1,
8812                        "min_supported": 2,
8813                        "max_supported": 1,
8814                    }),
8815                )],
8816                "version_marker_history_range_invalid",
8817            ),
8818        ] {
8819            let error = WorkflowState::new(
8820                history,
8821                "rust-workers".to_string(),
8822                JSON_CODEC.to_string(),
8823                None,
8824            )
8825            .expect_err("malformed history must fail");
8826            assert!(matches!(
8827                error,
8828                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
8829                    if actual == reason
8830            ));
8831        }
8832    }
8833
8834    #[test]
8835    fn duplicate_side_effects_and_version_markers_are_rejected() {
8836        let duplicate_side_effect = WorkflowState::new(
8837            vec![
8838                history_event(
8839                    "SideEffectRecorded",
8840                    json!({"sequence": 1, "result": {"codec": "json", "blob": "1"}}),
8841                ),
8842                history_event(
8843                    "SideEffectRecorded",
8844                    json!({"sequence": 1, "result": {"codec": "json", "blob": "2"}}),
8845                ),
8846            ],
8847            "rust-workers".to_string(),
8848            JSON_CODEC.to_string(),
8849            None,
8850        )
8851        .expect_err("duplicate side effect");
8852        assert!(matches!(
8853            duplicate_side_effect,
8854            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8855                if reason == "duplicate_side_effect_record"
8856        ));
8857
8858        let marker = |sequence| {
8859            history_event(
8860                "VersionMarkerRecorded",
8861                json!({
8862                    "sequence": sequence,
8863                    "change_id": "same-change",
8864                    "version": 1,
8865                    "min_supported": 1,
8866                    "max_supported": 1,
8867                }),
8868            )
8869        };
8870        let duplicate_marker = WorkflowState::new(
8871            vec![marker(1), marker(3)],
8872            "rust-workers".to_string(),
8873            JSON_CODEC.to_string(),
8874            None,
8875        )
8876        .expect_err("duplicate marker");
8877        assert!(matches!(
8878            duplicate_marker,
8879            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8880                if reason == "duplicate_version_marker"
8881        ));
8882    }
8883
8884    #[test]
8885    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
8886        fn worker(calls: Arc<AtomicUsize>) -> Worker {
8887            let client = Client::new("http://127.0.0.1:8080").expect("client");
8888            let mut worker = Worker::new(client, "rust-workers");
8889            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
8890                let calls = Arc::clone(&calls);
8891                async move {
8892                    let captured = ctx.side_effect(|| {
8893                        calls.fetch_add(1, Ordering::SeqCst);
8894                        "captured-once".to_string()
8895                    })?;
8896                    let version = ctx.get_version("cold-restart", 1, 2)?;
8897                    Ok(json!({"captured": captured, "version": version}))
8898                }
8899            });
8900            worker
8901        }
8902
8903        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
8904            WorkflowTask {
8905                task_id: "wft-side-effect-version".to_string(),
8906                workflow_id: Some("wf-side-effect-version".to_string()),
8907                run_id: Some("run-side-effect-version".to_string()),
8908                workflow_type: "rust.side-effect-version".to_string(),
8909                payload_codec: JSON_CODEC.to_string(),
8910                arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("arguments")),
8911                history_events,
8912                total_history_events: None,
8913                history_size_bytes: None,
8914                continue_as_new_recommended: None,
8915                history_budget_pressure: None,
8916                next_history_page_token: None,
8917                workflow_task_attempt: 1,
8918                workflow_signal_id: None,
8919                signal_name: None,
8920                signal_arguments: None,
8921                workflow_update_id: None,
8922                update_name: None,
8923                lease_owner: Some("rust-worker".to_string()),
8924            }
8925        }
8926
8927        let calls = Arc::new(AtomicUsize::new(0));
8928        let initial = worker(Arc::clone(&calls))
8929            .execute_workflow_task(task(Vec::new()))
8930            .expect("initial execution");
8931        assert_eq!(
8932            initial
8933                .iter()
8934                .map(|command| &command["type"])
8935                .collect::<Vec<_>>(),
8936            vec![
8937                "record_side_effect",
8938                "record_version_marker",
8939                "complete_workflow"
8940            ]
8941        );
8942        assert_eq!(calls.load(Ordering::SeqCst), 1);
8943
8944        let restarted = worker(Arc::clone(&calls));
8945        let replayed = restarted
8946            .execute_workflow_task(task(vec![
8947                history_event(
8948                    "SideEffectRecorded",
8949                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
8950                ),
8951                history_event(
8952                    "VersionMarkerRecorded",
8953                    json!({
8954                        "sequence": 2,
8955                        "change_id": "cold-restart",
8956                        "version": 2,
8957                        "min_supported": 1,
8958                        "max_supported": 2,
8959                    }),
8960                ),
8961            ]))
8962            .expect("cold replay");
8963        assert_eq!(replayed.len(), 1);
8964        assert_eq!(replayed[0]["type"], "complete_workflow");
8965        assert_eq!(calls.load(Ordering::SeqCst), 1);
8966    }
8967
8968    #[test]
8969    fn side_effect_replay_rejects_changed_rust_value_type() {
8970        let result = encode_value_envelope(&json!({"value": 42}), JSON_CODEC).expect("result");
8971        let ctx = workflow_context(vec![history_event(
8972            "SideEffectRecorded",
8973            json!({"sequence": 1, "result": result}),
8974        )]);
8975        let error = ctx
8976            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
8977            .expect_err("changed type must fail replay");
8978        assert!(matches!(
8979            error,
8980            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
8981                if reason == "side_effect_type_mismatch"
8982        ));
8983    }
8984
8985    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
8986        vec![
8987            history_event(
8988                "ActivityScheduled",
8989                json!({
8990                    "sequence": 1,
8991                    "activity_type": "flaky",
8992                    "activity_execution_id": "act-1",
8993                    "activity": {
8994                        "id": "act-1",
8995                        "sequence": 1,
8996                        "type": "flaky",
8997                        "queue": "critical-activities",
8998                        "execution_mode": null,
8999                        "retry_policy": {
9000                            "snapshot_version": 1,
9001                            "max_attempts": 3,
9002                            "backoff_seconds": [2, 4],
9003                            "start_to_close_timeout": 30,
9004                            "schedule_to_start_timeout": 5,
9005                            "schedule_to_close_timeout": 90,
9006                            "heartbeat_timeout": 10,
9007                            "non_retryable_error_types": ["PermanentError"]
9008                        }
9009                    }
9010                }),
9011            ),
9012            history_event(
9013                "ActivityStarted",
9014                json!({
9015                    "sequence": 1,
9016                    "activity_type": "flaky",
9017                    "activity_execution_id": "act-1",
9018                    "activity_attempt_id": "attempt-1",
9019                    "attempt_number": 1
9020                }),
9021            ),
9022            history_event(
9023                "ActivityRetryScheduled",
9024                json!({
9025                    "sequence": 1,
9026                    "activity_type": "flaky",
9027                    "activity_execution_id": "act-1",
9028                    "activity_attempt_id": "attempt-1",
9029                    "attempt_number": 1,
9030                    "retry_after_attempt": 1,
9031                    "retry_backoff_seconds": 2,
9032                    "failure_category": "activity",
9033                    "exception_type": "TransientError"
9034                }),
9035            ),
9036            history_event(
9037                "ActivityStarted",
9038                json!({
9039                    "sequence": 1,
9040                    "activity_type": "flaky",
9041                    "activity_execution_id": "act-1",
9042                    "activity_attempt_id": "attempt-2",
9043                    "attempt_number": 2
9044                }),
9045            ),
9046            history_event(
9047                "ActivityCompleted",
9048                json!({
9049                    "sequence": 1,
9050                    "activity_type": "flaky",
9051                    "activity_execution_id": "act-1",
9052                    "activity_attempt_id": "attempt-2",
9053                    "attempt_number": 2,
9054                    "payload_codec": "json",
9055                    "result": {"codec": "json", "blob": "{\"status\":\"recovered\"}"}
9056                }),
9057            ),
9058        ]
9059    }
9060
9061    fn retry_activity_options() -> ActivityOptions {
9062        ActivityOptions::new()
9063            .task_queue("critical-activities")
9064            .retry_policy(
9065                ActivityRetryPolicy::new(3)
9066                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
9067                    .non_retryable_error_type("PermanentError"),
9068            )
9069            .start_to_close_timeout(Duration::from_secs(30))
9070            .schedule_to_start_timeout(Duration::from_secs(5))
9071            .schedule_to_close_timeout(Duration::from_secs(90))
9072            .heartbeat_timeout(Duration::from_secs(10))
9073    }
9074
9075    #[test]
9076    fn fixed_avro_value_round_trips_json_values() {
9077        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9078        let envelope = PayloadEnvelope::avro(&value).expect("encode");
9079        assert_eq!(envelope.codec, DEFAULT_CODEC);
9080        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9081    }
9082
9083    #[tokio::test]
9084    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
9085        let client = Client::new("http://127.0.0.1:8080").expect("client");
9086        let mut worker = Worker::new(client, "rust-workers");
9087        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
9088        worker
9089            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
9090        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
9091            Ok(input)
9092        });
9093        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
9094            Ok(input)
9095        });
9096        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
9097            Ok(AvroValue::Array(
9098                ctx.wait_signal_avro_value("changed").await?,
9099            ))
9100        });
9101
9102        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
9103        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
9104
9105        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
9106        workflow.arguments = Some(envelope.clone());
9107        let commands = worker
9108            .execute_workflow_task(workflow)
9109            .expect("typed workflow task");
9110        assert_eq!(commands[0]["type"], "complete_workflow");
9111        assert_eq!(
9112            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9113                .expect("typed workflow result"),
9114            arguments
9115        );
9116
9117        let activity = ActivityTask {
9118            task_id: "activity-typed".to_string(),
9119            activity_attempt_id: Some("attempt-typed".to_string()),
9120            attempt_id: None,
9121            activity_type: "typed.activity".to_string(),
9122            payload_codec: DEFAULT_CODEC.to_string(),
9123            arguments: Some(envelope.clone()),
9124            attempt_number: 1,
9125            lease_owner: Some("rust-worker".to_string()),
9126        };
9127        assert_eq!(
9128            worker
9129                .execute_activity_task(activity)
9130                .await
9131                .expect("typed activity result"),
9132            arguments
9133        );
9134
9135        let query = QueryTask {
9136            query_task_id: "query-typed".to_string(),
9137            query_task_attempt: 1,
9138            lease_owner: Some("rust-worker".to_string()),
9139            workflow_id: Some("typed-1".to_string()),
9140            run_id: Some("run-typed".to_string()),
9141            workflow_type: "typed.echo".to_string(),
9142            query_name: "inspect".to_string(),
9143            payload_codec: DEFAULT_CODEC.to_string(),
9144            workflow_arguments: Some(
9145                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
9146                    .expect("workflow input"),
9147            ),
9148            query_arguments: Some(envelope.clone()),
9149            history_events: Vec::new(),
9150            history_export: None,
9151            run_status: Some("running".to_string()),
9152        };
9153        assert_eq!(
9154            worker
9155                .execute_query_task(query)
9156                .await
9157                .expect("typed query result"),
9158            arguments
9159        );
9160
9161        let mut update = workflow_task(
9162            "typed.echo",
9163            vec![history_event(
9164                "UpdateAccepted",
9165                json!({
9166                    "update_id": "update-typed",
9167                    "update_name": "replace",
9168                    "arguments": envelope.clone(),
9169                }),
9170            )],
9171            DEFAULT_CODEC,
9172        );
9173        update.workflow_update_id = Some("update-typed".to_string());
9174        update.update_name = Some("replace".to_string());
9175        let commands = worker
9176            .execute_workflow_task(update)
9177            .expect("typed update task");
9178        assert_eq!(commands[0]["type"], "complete_update");
9179        assert_eq!(
9180            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9181                .expect("typed update result"),
9182            arguments
9183        );
9184
9185        let mut signal = workflow_task(
9186            "typed.signal",
9187            vec![history_event(
9188                "SignalReceived",
9189                json!({
9190                    "signal_id": "signal-typed",
9191                    "signal_name": "changed",
9192                    "arguments": envelope.clone(),
9193                }),
9194            )],
9195            DEFAULT_CODEC,
9196        );
9197        signal.workflow_signal_id = Some("signal-typed".to_string());
9198        signal.signal_name = Some("changed".to_string());
9199        signal.signal_arguments = Some(envelope);
9200        let commands = worker
9201            .execute_workflow_task(signal)
9202            .expect("typed signal resume");
9203        assert_eq!(
9204            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
9205                .expect("typed signal result"),
9206            arguments
9207        );
9208    }
9209
9210    #[tokio::test]
9211    async fn typed_helpers_never_parse_json_inspection_projection() {
9212        let collision_values = projection_collision_probe();
9213        let expected = AvroValue::Array(collision_values.clone());
9214        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
9215
9216        let activity_context = workflow_context_with_codec(
9217            vec![history_event(
9218                "ActivityCompleted",
9219                json!({
9220                    "sequence": 1,
9221                    "activity_type": "collision.activity",
9222                    "payload_codec": DEFAULT_CODEC,
9223                    "result": envelope.clone(),
9224                }),
9225            )],
9226            DEFAULT_CODEC,
9227        );
9228        assert_eq!(
9229            activity_context
9230                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
9231                .await
9232                .expect("typed activity collision result"),
9233            expected
9234        );
9235
9236        let signal_context = workflow_context_with_codec(
9237            vec![
9238                history_event(
9239                    "SignalWaitOpened",
9240                    json!({"sequence": 1, "signal_name": "collision"}),
9241                ),
9242                history_event(
9243                    "SignalApplied",
9244                    json!({
9245                        "sequence": 1,
9246                        "signal_name": "collision",
9247                        "payload_codec": DEFAULT_CODEC,
9248                        "value": envelope.clone(),
9249                    }),
9250                ),
9251            ],
9252            DEFAULT_CODEC,
9253        );
9254        assert_eq!(
9255            signal_context
9256                .wait_signal_avro_value("collision")
9257                .await
9258                .expect("typed signal collision arguments"),
9259            collision_values
9260        );
9261
9262        let child_context = workflow_context_with_codec(
9263            vec![
9264                history_event(
9265                    "ChildWorkflowScheduled",
9266                    json!({
9267                        "sequence": 1,
9268                        "child_workflow_instance_id": "collision-child",
9269                        "child_workflow_run_id": "collision-run",
9270                        "child_workflow_type": "collision.child",
9271                    }),
9272                ),
9273                history_event(
9274                    "ChildRunCompleted",
9275                    json!({
9276                        "sequence": 1,
9277                        "child_workflow_instance_id": "collision-child",
9278                        "child_workflow_run_id": "collision-run",
9279                        "child_workflow_type": "collision.child",
9280                        "payload_codec": DEFAULT_CODEC,
9281                        "result": envelope,
9282                    }),
9283                ),
9284            ],
9285            DEFAULT_CODEC,
9286        );
9287        let child = child_context
9288            .start_child_workflow_avro_value(
9289                "collision.child",
9290                ChildWorkflowOptions::new("collision-workers"),
9291                AvroValue::Array(Vec::new()),
9292            )
9293            .await
9294            .expect("typed child collision result");
9295        assert_eq!(child.result, expected);
9296    }
9297
9298    #[tokio::test]
9299    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
9300        let client = Client::new("http://127.0.0.1:8080").expect("client");
9301        let mut worker = Worker::new(client, "rust-workers");
9302        worker.register_replayed_workflow_avro_value(
9303            "typed.replayed",
9304            || (),
9305            |_ctx, input, _state| async move { Ok(input) },
9306        );
9307        worker.register_replayed_query_avro_value::<(), _, _>(
9308            "typed.replayed",
9309            "inspect",
9310            |ctx, _state, args| async move {
9311                let mut signals = ctx.signals_avro_value("collision");
9312                let signal = signals
9313                    .pop()
9314                    .map(AvroValue::Array)
9315                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
9316                Ok(AvroValue::Array(vec![
9317                    ctx.workflow_input_avro_value().clone(),
9318                    signal,
9319                    args,
9320                ]))
9321            },
9322        );
9323        let arguments = AvroValue::Array(projection_collision_probe());
9324        let signal_arguments =
9325            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
9326        let task = QueryTask {
9327            query_task_id: "query-typed-replay".to_string(),
9328            query_task_attempt: 1,
9329            lease_owner: Some("rust-worker".to_string()),
9330            workflow_id: Some("typed-replay".to_string()),
9331            run_id: Some("run-typed-replay".to_string()),
9332            workflow_type: "typed.replayed".to_string(),
9333            query_name: "inspect".to_string(),
9334            payload_codec: DEFAULT_CODEC.to_string(),
9335            workflow_arguments: Some(
9336                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
9337            ),
9338            query_arguments: Some(
9339                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
9340            ),
9341            history_events: vec![history_event(
9342                "SignalReceived",
9343                json!({
9344                    "signal_id": "collision-signal",
9345                    "signal_name": "collision",
9346                    "workflow_sequence": 1,
9347                    "payload_codec": DEFAULT_CODEC,
9348                    "arguments": signal_arguments,
9349                }),
9350            )],
9351            history_export: None,
9352            run_status: Some("completed".to_string()),
9353        };
9354
9355        assert_eq!(
9356            worker
9357                .execute_query_task(task)
9358                .await
9359                .expect("typed replay query"),
9360            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
9361        );
9362    }
9363
9364    #[test]
9365    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
9366        let value = BTreeMap::from([(1_i32, "integer key")]);
9367        let error = PayloadEnvelope::avro(&value)
9368            .expect_err("integer map keys must fail")
9369            .to_string();
9370
9371        assert!(error.contains("invalid_map_key"));
9372    }
9373
9374    #[test]
9375    fn json_codec_remains_plain_json() {
9376        let value = json!({"greeting": "hello", "count": 3, "ok": true});
9377        let envelope = PayloadEnvelope::json(&value).expect("encode");
9378
9379        assert_eq!(envelope.codec, JSON_CODEC);
9380        assert_eq!(envelope.blob, serde_json::to_string(&value).expect("json"));
9381        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
9382    }
9383
9384    #[test]
9385    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
9386        let envelope = PayloadEnvelope {
9387            codec: DEFAULT_CODEC.to_string(),
9388            blob: BASE64.encode([0x01]),
9389        };
9390
9391        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
9392        assert!(error.to_string().contains("invalid_payload_framing"));
9393    }
9394
9395    #[test]
9396    fn workflow_context_schedules_activity_until_completion_is_in_history() {
9397        let ctx = WorkflowContext {
9398            state: Arc::new(Mutex::new(
9399                WorkflowState::new_with_identity(
9400                    Vec::new(),
9401                    Some("wf-parent".to_string()),
9402                    Some("run-parent".to_string()),
9403                    "rust-workers".to_string(),
9404                    DEFAULT_CODEC.to_string(),
9405                    None,
9406                )
9407                .expect("workflow state"),
9408            )),
9409        };
9410
9411        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
9412        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9413        assert!(matches!(
9414            call.as_mut().poll(&mut task_context),
9415            Poll::Pending
9416        ));
9417
9418        let commands = ctx.take_commands().expect("commands");
9419        assert_eq!(commands[0]["type"], "schedule_activity");
9420        assert_eq!(commands[0]["activity_type"], "hello.activity");
9421    }
9422
9423    #[test]
9424    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
9425        let ctx = workflow_context(Vec::new());
9426        let options = ActivityOptions::new()
9427            .task_queue("payments")
9428            .retry_policy(
9429                ActivityRetryPolicy::new(4)
9430                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
9431                    .non_retryable_error_type("ValidationError"),
9432            )
9433            .start_to_close_timeout(Duration::from_secs(120))
9434            .schedule_to_start_timeout(Duration::from_secs(10))
9435            .schedule_to_close_timeout(Duration::from_secs(300))
9436            .heartbeat_timeout(Duration::from_secs(15));
9437        let mut call = Box::pin(ctx.activity_with_options(
9438            "charge-card",
9439            options,
9440            json!([{"order_id": "o-1"}]),
9441        ));
9442        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9443
9444        assert!(matches!(
9445            call.as_mut().poll(&mut task_context),
9446            Poll::Pending
9447        ));
9448        assert!(matches!(
9449            call.as_mut().poll(&mut task_context),
9450            Poll::Pending
9451        ));
9452
9453        let commands = ctx.take_commands().expect("activity command");
9454        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
9455        assert_eq!(commands[0]["queue"], "payments");
9456        assert_eq!(
9457            commands[0]["retry_policy"],
9458            json!({
9459                "max_attempts": 4,
9460                "backoff_seconds": [1, 3, 9],
9461                "non_retryable_error_types": ["ValidationError"],
9462            })
9463        );
9464        assert_eq!(commands[0]["start_to_close_timeout"], 120);
9465        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
9466        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
9467        assert_eq!(commands[0]["heartbeat_timeout"], 15);
9468    }
9469
9470    #[test]
9471    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
9472        let ctx = workflow_context(Vec::new());
9473        let options = ActivityOptions::new().retry_policy(
9474            ActivityRetryPolicy::new(3)
9475                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
9476        );
9477        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9478        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9479
9480        assert!(matches!(
9481            call.as_mut().poll(&mut task_context),
9482            Poll::Pending
9483        ));
9484        assert_eq!(
9485            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
9486            json!([1, 2])
9487        );
9488    }
9489
9490    #[test]
9491    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
9492        let cases = [
9493            (
9494                ActivityOptions::new().task_queue("  "),
9495                ActivityOptionsErrorKind::EmptyTaskQueue,
9496            ),
9497            (
9498                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
9499                ActivityOptionsErrorKind::EmptyRetryPolicy,
9500            ),
9501            (
9502                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
9503                ActivityOptionsErrorKind::InvalidMaxAttempts,
9504            ),
9505            (
9506                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
9507                    max_attempts: None,
9508                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
9509                    non_retryable_error_types: Vec::new(),
9510                }),
9511                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
9512            ),
9513            (
9514                ActivityOptions::new().retry_policy(
9515                    ActivityRetryPolicy::new(2)
9516                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
9517                ),
9518                ActivityOptionsErrorKind::TooManyBackoffIntervals,
9519            ),
9520            (
9521                ActivityOptions::new().retry_policy(
9522                    ActivityRetryPolicy::new(2).exponential_backoff(
9523                        Duration::from_secs(1),
9524                        0,
9525                        None,
9526                    ),
9527                ),
9528                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
9529            ),
9530            (
9531                ActivityOptions::new()
9532                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
9533                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
9534            ),
9535            (
9536                ActivityOptions::new().retry_policy(
9537                    ActivityRetryPolicy::new(10_002).exponential_backoff(
9538                        Duration::from_secs(1),
9539                        1,
9540                        None,
9541                    ),
9542                ),
9543                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
9544            ),
9545            (
9546                ActivityOptions::new().retry_policy(
9547                    ActivityRetryPolicy::new(2)
9548                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
9549                ),
9550                ActivityOptionsErrorKind::BackoffOverflow,
9551            ),
9552        ];
9553
9554        for (options, expected_kind) in cases {
9555            let ctx = workflow_context(Vec::new());
9556            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
9557            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9558            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
9559                call.as_mut().poll(&mut task_context)
9560            else {
9561                panic!("expected typed activity validation error");
9562            };
9563            assert_eq!(error.kind, expected_kind);
9564            assert!(ctx.take_commands().expect("commands").is_empty());
9565        }
9566    }
9567
9568    #[test]
9569    fn activity_options_validate_positive_and_ordered_timeouts() {
9570        let zero_timeout_cases = [
9571            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
9572            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
9573            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
9574            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
9575        ];
9576        for options in zero_timeout_cases {
9577            assert_eq!(
9578                options.validate().expect_err("zero timeout").kind,
9579                ActivityOptionsErrorKind::TimeoutNotPositive
9580            );
9581        }
9582
9583        let ordering_cases = [
9584            ActivityOptions::new()
9585                .heartbeat_timeout(Duration::from_secs(11))
9586                .start_to_close_timeout(Duration::from_secs(10)),
9587            ActivityOptions::new()
9588                .start_to_close_timeout(Duration::from_secs(31))
9589                .schedule_to_close_timeout(Duration::from_secs(30)),
9590            ActivityOptions::new()
9591                .schedule_to_start_timeout(Duration::from_secs(31))
9592                .schedule_to_close_timeout(Duration::from_secs(30)),
9593        ];
9594        for options in ordering_cases {
9595            assert_eq!(
9596                options.validate().expect_err("timeout order").kind,
9597                ActivityOptionsErrorKind::TimeoutOrder
9598            );
9599        }
9600
9601        assert_eq!(
9602            ActivityOptions::new()
9603                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
9604                .validate()
9605                .expect_err("protocol integer overflow")
9606                .kind,
9607            ActivityOptionsErrorKind::TimeoutOverflow
9608        );
9609    }
9610
9611    #[test]
9612    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
9613        let ctx = workflow_context(completed_retry_activity_history());
9614        let mut call =
9615            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9616        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9617
9618        assert!(matches!(
9619            call.as_mut().poll(&mut task_context),
9620            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9621        ));
9622        assert!(ctx.take_commands().expect("commands").is_empty());
9623        ctx.ensure_history_consumed().expect("history consumed");
9624    }
9625
9626    #[test]
9627    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
9628        let mut options = retry_activity_options();
9629        options
9630            .retry_policy
9631            .as_mut()
9632            .expect("retry policy")
9633            .non_retryable_error_types
9634            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
9635
9636        let new_ctx = workflow_context(Vec::new());
9637        let mut new_call =
9638            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
9639        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9640        assert!(matches!(
9641            new_call.as_mut().poll(&mut task_context),
9642            Poll::Pending
9643        ));
9644        let commands = new_ctx.take_commands().expect("commands");
9645        assert_eq!(commands.len(), 1);
9646        assert_eq!(
9647            commands[0]["retry_policy"]["non_retryable_error_types"],
9648            json!(["PermanentError"])
9649        );
9650
9651        let replay_ctx = workflow_context(completed_retry_activity_history());
9652        let mut replay_call =
9653            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
9654        assert!(matches!(
9655            replay_call.as_mut().poll(&mut task_context),
9656            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9657        ));
9658        assert!(replay_ctx.take_commands().expect("commands").is_empty());
9659        replay_ctx
9660            .ensure_history_consumed()
9661            .expect("history consumed");
9662    }
9663
9664    #[test]
9665    fn replayed_intermediate_retry_remains_pending_across_restarts() {
9666        let history = completed_retry_activity_history()
9667            .into_iter()
9668            .take(3)
9669            .collect::<Vec<_>>();
9670
9671        for _restart in 0..2 {
9672            let ctx = workflow_context(history.clone());
9673            let mut call =
9674                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9675            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9676            assert!(matches!(
9677                call.as_mut().poll(&mut task_context),
9678                Poll::Pending
9679            ));
9680            assert!(ctx.take_commands().expect("commands").is_empty());
9681        }
9682    }
9683
9684    #[test]
9685    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
9686        let mut changed_queue = retry_activity_options();
9687        changed_queue.task_queue = Some("different-queue".to_string());
9688
9689        let mut changed_max_attempts = retry_activity_options();
9690        let retry_policy = changed_max_attempts
9691            .retry_policy
9692            .as_mut()
9693            .expect("retry policy");
9694        retry_policy.max_attempts = Some(4);
9695
9696        let mut changed_backoff = retry_activity_options();
9697        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
9698        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
9699            Duration::from_secs(3),
9700            Duration::from_secs(4),
9701        ]));
9702
9703        let mut changed_non_retryable_types = retry_activity_options();
9704        let retry_policy = changed_non_retryable_types
9705            .retry_policy
9706            .as_mut()
9707            .expect("retry policy");
9708        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
9709
9710        let mut changed_start_to_close = retry_activity_options();
9711        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
9712        let mut changed_schedule_to_start = retry_activity_options();
9713        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
9714        let mut changed_schedule_to_close = retry_activity_options();
9715        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
9716        let mut changed_heartbeat = retry_activity_options();
9717        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
9718
9719        let cases = [
9720            (changed_queue, "activity_task_queue_mismatch"),
9721            (changed_max_attempts, "activity_retry_policy_mismatch"),
9722            (changed_backoff, "activity_retry_policy_mismatch"),
9723            (
9724                changed_non_retryable_types,
9725                "activity_retry_policy_mismatch",
9726            ),
9727            (changed_start_to_close, "activity_retry_policy_mismatch"),
9728            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
9729            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
9730            (changed_heartbeat, "activity_retry_policy_mismatch"),
9731        ];
9732
9733        for (options, expected_reason) in cases {
9734            let ctx = workflow_context(completed_retry_activity_history());
9735            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
9736            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9737            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9738                call.as_mut().poll(&mut task_context)
9739            else {
9740                panic!("changed activity options must fail replay");
9741            };
9742            assert_eq!(failure.reason, expected_reason);
9743            assert_eq!(failure.sequence, Some(1));
9744            assert!(ctx.take_commands().expect("commands").is_empty());
9745        }
9746    }
9747
9748    #[test]
9749    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
9750        let cases = [
9751            (
9752                "execution_mode",
9753                json!("local"),
9754                "activity_execution_mode_mismatch",
9755            ),
9756            (
9757                "snapshot_version",
9758                json!(2),
9759                "activity_retry_policy_mismatch",
9760            ),
9761        ];
9762
9763        for (field, value, expected_reason) in cases {
9764            let mut history = completed_retry_activity_history();
9765            let activity = history[0].payload["activity"]
9766                .as_object_mut()
9767                .expect("activity snapshot");
9768            if field == "execution_mode" {
9769                activity.insert(field.to_string(), value);
9770            } else {
9771                activity["retry_policy"]
9772                    .as_object_mut()
9773                    .expect("retry snapshot")
9774                    .insert(field.to_string(), value);
9775            }
9776
9777            let ctx = workflow_context(history);
9778            let mut call =
9779                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9780            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9781            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
9782                call.as_mut().poll(&mut task_context)
9783            else {
9784                panic!("changed {field} must fail replay");
9785            };
9786            assert_eq!(failure.reason, expected_reason);
9787            assert_eq!(failure.sequence, Some(1));
9788            assert!(ctx.take_commands().expect("commands").is_empty());
9789        }
9790    }
9791
9792    #[test]
9793    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
9794        let mut history = completed_retry_activity_history();
9795        let activity = history[0].payload["activity"]
9796            .as_object_mut()
9797            .expect("activity snapshot");
9798        activity.remove("execution_mode");
9799        activity.remove("retry_policy");
9800
9801        let mut current = retry_activity_options();
9802        current.start_to_close_timeout = Some(Duration::from_secs(45));
9803        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
9804        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
9805        current.heartbeat_timeout = Some(Duration::from_secs(12));
9806
9807        let ctx = workflow_context(history);
9808        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
9809        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9810        assert!(matches!(
9811            call.as_mut().poll(&mut task_context),
9812            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
9813        ));
9814        assert!(ctx.take_commands().expect("commands").is_empty());
9815        ctx.ensure_history_consumed().expect("history consumed");
9816    }
9817
9818    #[test]
9819    fn terminal_activity_failed_after_start_returns_typed_failure() {
9820        let history = vec![
9821            history_event(
9822                "ActivityScheduled",
9823                json!({
9824                    "sequence": 1,
9825                    "activity_type": "flaky",
9826                    "activity_execution_id": "act-terminal",
9827                    "activity": {
9828                        "id": "act-terminal",
9829                        "sequence": 1,
9830                        "type": "flaky",
9831                        "queue": "critical-activities",
9832                        "retry_policy": {
9833                            "snapshot_version": 1,
9834                            "max_attempts": 3,
9835                            "backoff_seconds": [2, 4],
9836                            "non_retryable_error_types": ["PermanentError"]
9837                        }
9838                    }
9839                }),
9840            ),
9841            history_event(
9842                "ActivityStarted",
9843                json!({
9844                    "sequence": 1,
9845                    "activity_type": "flaky",
9846                    "activity_execution_id": "act-terminal",
9847                    "activity_attempt_id": "attempt-1",
9848                    "attempt_number": 1
9849                }),
9850            ),
9851            history_event(
9852                "ActivityFailed",
9853                json!({
9854                    "sequence": 1,
9855                    "activity_type": "flaky",
9856                    "activity_execution_id": "act-terminal",
9857                    "activity_attempt_id": "attempt-1",
9858                    "attempt_number": 1,
9859                    "failure_id": "failure-terminal",
9860                    "failure_category": "activity",
9861                    "exception_type": "PermanentError",
9862                    "message": "cannot retry",
9863                    "non_retryable": true
9864                }),
9865            ),
9866        ];
9867        let ctx = workflow_context(history);
9868        let mut call =
9869            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
9870        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9871
9872        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9873            call.as_mut().poll(&mut task_context)
9874        else {
9875            panic!("terminal ActivityFailed must settle the activity future");
9876        };
9877        assert_eq!(failure.kind, ActivityFailureKind::Failed);
9878        assert_eq!(
9879            failure.activity_execution_id.as_deref(),
9880            Some("act-terminal")
9881        );
9882        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
9883        assert!(failure.non_retryable);
9884        assert!(ctx.take_commands().expect("commands").is_empty());
9885        ctx.ensure_history_consumed().expect("history consumed");
9886    }
9887
9888    #[test]
9889    fn activity_terminal_events_return_machine_readable_failures() {
9890        let cases = [
9891            (
9892                "ActivityFailed",
9893                json!({
9894                    "sequence": 1,
9895                    "activity_type": "charge-card",
9896                    "activity_execution_id": "act-1",
9897                    "activity_attempt_id": "attempt-2",
9898                    "attempt_number": 2,
9899                    "failure_id": "failure-1",
9900                    "failure_category": "activity",
9901                    "exception_type": "PaymentDeclined",
9902                    "exception_class": "payments.PaymentDeclined",
9903                    "message": "card declined",
9904                    "non_retryable": true
9905                }),
9906                ActivityFailureKind::Failed,
9907                "activity",
9908            ),
9909            (
9910                "ActivityCancelled",
9911                json!({
9912                    "sequence": 1,
9913                    "activity_type": "charge-card",
9914                    "activity_execution_id": "act-1",
9915                    "activity_attempt_id": "attempt-1"
9916                }),
9917                ActivityFailureKind::Cancelled,
9918                "cancelled",
9919            ),
9920        ];
9921
9922        for (event_type, payload, expected_kind, expected_reason) in cases {
9923            let ctx = workflow_context(vec![history_event(event_type, payload)]);
9924            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
9925            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9926            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9927                call.as_mut().poll(&mut task_context)
9928            else {
9929                panic!("expected terminal activity failure");
9930            };
9931            assert_eq!(failure.kind, expected_kind);
9932            assert_eq!(failure.reason, expected_reason);
9933            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
9934            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
9935        }
9936    }
9937
9938    #[test]
9939    fn every_activity_timeout_class_is_typed() {
9940        for timeout_kind in [
9941            "start_to_close",
9942            "schedule_to_start",
9943            "schedule_to_close",
9944            "heartbeat",
9945        ] {
9946            let ctx = workflow_context(vec![history_event(
9947                "ActivityTimedOut",
9948                json!({
9949                    "sequence": 1,
9950                    "activity_type": "slow",
9951                    "activity_execution_id": "act-timeout",
9952                    "activity_attempt_id": "attempt-timeout",
9953                    "failure_category": "timeout",
9954                    "timeout_kind": timeout_kind,
9955                    "message": "deadline expired"
9956                }),
9957            )]);
9958            let mut call = Box::pin(ctx.activity("slow", json!([])));
9959            let mut task_context = TaskContext::from_waker(noop_waker_ref());
9960            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
9961                call.as_mut().poll(&mut task_context)
9962            else {
9963                panic!("expected timeout failure");
9964            };
9965            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
9966            assert_eq!(failure.reason, timeout_kind);
9967            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
9968            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
9969        }
9970    }
9971
9972    #[test]
9973    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
9974        let ctx = workflow_context(Vec::new());
9975        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
9976        let mut task_context = TaskContext::from_waker(noop_waker_ref());
9977
9978        assert!(matches!(
9979            sleep.as_mut().poll(&mut task_context),
9980            Poll::Pending
9981        ));
9982        assert!(matches!(
9983            sleep.as_mut().poll(&mut task_context),
9984            Poll::Pending
9985        ));
9986
9987        let commands = ctx.take_commands().expect("timer command");
9988        assert_eq!(
9989            commands,
9990            vec![json!({
9991                "type": "start_timer",
9992                "delay_seconds": 2,
9993            })]
9994        );
9995    }
9996
9997    #[test]
9998    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
9999        let history = vec![
10000            history_event(
10001                "TimerScheduled",
10002                json!({
10003                    "sequence": 1,
10004                    "timer_id": "timer-1",
10005                    "delay_seconds": 5,
10006                    "fire_at": "2026-07-11T12:00:05Z",
10007                }),
10008            ),
10009            history_event(
10010                "TimerFired",
10011                json!({
10012                    "sequence": 1,
10013                    "timer_id": "timer-1",
10014                    "delay_seconds": 5,
10015                    "fire_at": "2026-07-11T12:00:05Z",
10016                    "fired_at": "2026-07-11T12:00:05Z",
10017                }),
10018            ),
10019        ];
10020
10021        for _restart in 0..2 {
10022            let ctx = workflow_context(history.clone());
10023            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
10024            let mut task_context = TaskContext::from_waker(noop_waker_ref());
10025            assert!(matches!(
10026                sleep.as_mut().poll(&mut task_context),
10027                Poll::Ready(Ok(()))
10028            ));
10029            assert!(ctx.take_commands().expect("commands").is_empty());
10030            ctx.ensure_history_consumed().expect("history consumed");
10031        }
10032    }
10033
10034    #[test]
10035    fn workflow_sleep_rejects_changed_delay_during_replay() {
10036        let ctx = workflow_context(vec![
10037            history_event(
10038                "TimerScheduled",
10039                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10040            ),
10041            history_event(
10042                "TimerFired",
10043                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10044            ),
10045        ]);
10046        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
10047        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10048
10049        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10050            sleep.as_mut().poll(&mut task_context)
10051        else {
10052            panic!("changed timer delay must be rejected");
10053        };
10054        assert_eq!(failure.reason, "timer_delay_mismatch");
10055        assert_eq!(failure.sequence, Some(1));
10056    }
10057
10058    #[test]
10059    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
10060        let lone_fire = WorkflowState::new(
10061            vec![history_event(
10062                "TimerFired",
10063                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10064            )],
10065            "rust-workers".to_string(),
10066            JSON_CODEC.to_string(),
10067            None,
10068        )
10069        .expect_err("TimerFired requires TimerScheduled");
10070        assert!(matches!(
10071            lone_fire,
10072            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10073                if reason == "timer_schedule_missing_or_duplicate"
10074        ));
10075
10076        let wrong_identity = WorkflowState::new(
10077            vec![
10078                history_event(
10079                    "TimerScheduled",
10080                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10081                ),
10082                history_event(
10083                    "TimerFired",
10084                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10085                ),
10086            ],
10087            "rust-workers".to_string(),
10088            JSON_CODEC.to_string(),
10089            None,
10090        )
10091        .expect_err("fire must match scheduled timer identity");
10092        assert!(matches!(
10093            wrong_identity,
10094            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10095                if reason == "timer_identity_mismatch"
10096        ));
10097
10098        let duplicate_fire = WorkflowState::new(
10099            vec![
10100                history_event(
10101                    "TimerScheduled",
10102                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10103                ),
10104                history_event(
10105                    "TimerFired",
10106                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10107                ),
10108                history_event(
10109                    "TimerFired",
10110                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10111                ),
10112            ],
10113            "rust-workers".to_string(),
10114            JSON_CODEC.to_string(),
10115            None,
10116        )
10117        .expect_err("a durable timer cannot fire twice");
10118        assert!(matches!(
10119            duplicate_fire,
10120            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10121                if reason == "duplicate_timer_fire"
10122        ));
10123
10124        let wrong_fired_delay = WorkflowState::new(
10125            vec![
10126                history_event(
10127                    "TimerScheduled",
10128                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10129                ),
10130                history_event(
10131                    "TimerFired",
10132                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
10133                ),
10134            ],
10135            "rust-workers".to_string(),
10136            JSON_CODEC.to_string(),
10137            None,
10138        )
10139        .expect_err("timer schedule and fire delays must agree");
10140        assert!(matches!(
10141            wrong_fired_delay,
10142            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10143                if reason == "timer_history_delay_mismatch"
10144        ));
10145    }
10146
10147    #[test]
10148    fn replay_rejects_activity_moved_before_recorded_timer() {
10149        let ctx = workflow_context(vec![
10150            history_event(
10151                "TimerScheduled",
10152                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10153            ),
10154            history_event(
10155                "TimerFired",
10156                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10157            ),
10158            history_event(
10159                "ActivityCompleted",
10160                json!({
10161                    "sequence": 2,
10162                    "activity_type": "after-timer",
10163                    "payload_codec": "json",
10164                    "result": {"codec": "json", "blob": "\"done\""},
10165                }),
10166            ),
10167        ]);
10168        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
10169        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10170
10171        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10172            activity.as_mut().poll(&mut task_context)
10173        else {
10174            panic!("reordered durable command must be rejected");
10175        };
10176        assert_eq!(failure.reason, "recorded_command_mismatch");
10177        assert_eq!(failure.sequence, Some(1));
10178        assert_eq!(failure.expected.as_deref(), Some("timer"));
10179        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
10180    }
10181
10182    #[test]
10183    fn workflow_context_emits_a_typed_named_signal_wait() {
10184        let ctx = workflow_context(Vec::new());
10185        let mut signal = Box::pin(ctx.wait_signal("finish"));
10186        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10187
10188        assert!(matches!(
10189            signal.as_mut().poll(&mut task_context),
10190            Poll::Pending
10191        ));
10192        assert_eq!(
10193            ctx.take_commands().expect("signal-wait command"),
10194            vec![json!({
10195                "type": "open_signal_wait",
10196                "signal_name": "finish",
10197            })]
10198        );
10199    }
10200
10201    #[test]
10202    fn condition_wait_history_does_not_resolve_a_typed_signal_wait() {
10203        let ctx = workflow_context(vec![
10204            history_event(
10205                "ConditionWaitOpened",
10206                json!({"sequence": 1, "condition_key": "signal:finish"}),
10207            ),
10208            history_event(
10209                "ConditionWaitSatisfied",
10210                json!({"sequence": 1, "condition_key": "signal:finish"}),
10211            ),
10212            history_event(
10213                "SignalReceived",
10214                json!({"signal_name": "finish", "arguments": []}),
10215            ),
10216        ]);
10217        let mut signal = Box::pin(ctx.wait_signal("finish"));
10218        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10219
10220        assert!(matches!(
10221            signal.as_mut().poll(&mut task_context),
10222            Poll::Pending
10223        ));
10224        assert_eq!(
10225            ctx.take_commands().expect("typed signal-wait command"),
10226            vec![json!({
10227                "type": "open_signal_wait",
10228                "signal_name": "finish",
10229            })]
10230        );
10231    }
10232
10233    #[test]
10234    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
10235        let signal_then_timer = vec![
10236            history_event(
10237                "SignalWaitOpened",
10238                json!({"sequence": 1, "signal_name": "go"}),
10239            ),
10240            history_event(
10241                "SignalApplied",
10242                json!({
10243                    "sequence": 1,
10244                    "signal_name": "go",
10245                    "value": {"codec": "json", "blob": "[\"now\"]"},
10246                }),
10247            ),
10248            history_event(
10249                "TimerScheduled",
10250                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10251            ),
10252            history_event(
10253                "TimerFired",
10254                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
10255            ),
10256        ];
10257
10258        let ctx = workflow_context(signal_then_timer.clone());
10259        let mut signal = Box::pin(ctx.wait_signal("go"));
10260        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10261        assert!(matches!(
10262            signal.as_mut().poll(&mut task_context),
10263            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
10264        ));
10265        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
10266        assert!(matches!(
10267            timer.as_mut().poll(&mut task_context),
10268            Poll::Ready(Ok(()))
10269        ));
10270        ctx.ensure_history_consumed()
10271            .expect("signal and timer history consumed in order");
10272
10273        let reordered = workflow_context(signal_then_timer);
10274        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
10275        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10276            timer_first.as_mut().poll(&mut task_context)
10277        else {
10278            panic!("timer cannot consume signal-wait-first history");
10279        };
10280        assert_eq!(failure.reason, "recorded_command_mismatch");
10281        assert_eq!(failure.sequence, Some(1));
10282        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
10283
10284        let timer_then_signal = vec![
10285            history_event(
10286                "TimerScheduled",
10287                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10288            ),
10289            history_event(
10290                "TimerFired",
10291                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10292            ),
10293            history_event(
10294                "SignalWaitOpened",
10295                json!({"sequence": 2, "signal_name": "go"}),
10296            ),
10297            history_event(
10298                "SignalApplied",
10299                json!({
10300                    "sequence": 2,
10301                    "signal_name": "go",
10302                    "value": {"codec": "json", "blob": "[]"},
10303                }),
10304            ),
10305        ];
10306        let reordered = workflow_context(timer_then_signal);
10307        let mut signal_first = Box::pin(reordered.wait_signal("go"));
10308        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
10309            signal_first.as_mut().poll(&mut task_context)
10310        else {
10311            panic!("signal wait cannot consume timer-first history");
10312        };
10313        assert_eq!(failure.reason, "recorded_command_mismatch");
10314        assert_eq!(failure.sequence, Some(1));
10315        assert_eq!(failure.expected.as_deref(), Some("timer"));
10316    }
10317
10318    #[test]
10319    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
10320        let duplicate_timer = WorkflowState::new(
10321            vec![
10322                history_event(
10323                    "TimerScheduled",
10324                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10325                ),
10326                history_event(
10327                    "TimerScheduled",
10328                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
10329                ),
10330            ],
10331            "rust-workers".to_string(),
10332            JSON_CODEC.to_string(),
10333            None,
10334        )
10335        .expect_err("one workflow sequence cannot schedule two timers");
10336        assert!(matches!(
10337            duplicate_timer,
10338            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10339                if reason == "timer_schedule_missing_or_duplicate"
10340        ));
10341
10342        let colliding_kinds = WorkflowState::new(
10343            vec![
10344                history_event(
10345                    "TimerScheduled",
10346                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10347                ),
10348                history_event(
10349                    "ActivityCompleted",
10350                    json!({"sequence": 1, "activity_type": "same-sequence"}),
10351                ),
10352            ],
10353            "rust-workers".to_string(),
10354            JSON_CODEC.to_string(),
10355            None,
10356        )
10357        .expect_err("one workflow sequence cannot identify two command kinds");
10358        assert!(matches!(
10359            colliding_kinds,
10360            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10361                if reason == "durable_command_sequence_collision"
10362        ));
10363
10364        let duplicate_signal_wait = WorkflowState::new(
10365            vec![
10366                history_event(
10367                    "SignalWaitOpened",
10368                    json!({"sequence": 1, "signal_name": "go"}),
10369                ),
10370                history_event(
10371                    "SignalWaitOpened",
10372                    json!({"sequence": 1, "signal_name": "go"}),
10373                ),
10374            ],
10375            "rust-workers".to_string(),
10376            JSON_CODEC.to_string(),
10377            None,
10378        )
10379        .expect_err("one workflow sequence cannot open two signal waits");
10380        assert!(matches!(
10381            duplicate_signal_wait,
10382            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10383                if reason == "signal_wait_open_missing_or_duplicate"
10384        ));
10385    }
10386
10387    #[test]
10388    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
10389        let result = encode_value_envelope(&json!({"captured": true}), JSON_CODEC)
10390            .expect("side-effect result");
10391        let ctx = workflow_context(vec![history_event(
10392            "SideEffectRecorded",
10393            json!({"sequence": 99, "result": result}),
10394        )]);
10395
10396        let replayed: Value = ctx
10397            .side_effect(|| panic!("recorded side effect must not run"))
10398            .expect("positive global workflow sequence is valid");
10399        assert_eq!(replayed, json!({"captured": true}));
10400        ctx.ensure_history_consumed().expect("history consumed");
10401    }
10402
10403    #[test]
10404    fn workflow_history_rejects_zero_and_descending_command_sequences() {
10405        let result =
10406            encode_value_envelope(&json!("captured"), JSON_CODEC).expect("side-effect result");
10407        let zero = WorkflowState::new(
10408            vec![history_event(
10409                "SideEffectRecorded",
10410                json!({"sequence": 0, "result": result.clone()}),
10411            )],
10412            "rust-workers".to_string(),
10413            JSON_CODEC.to_string(),
10414            None,
10415        )
10416        .expect_err("durable command sequences must be positive");
10417        assert!(matches!(
10418            zero,
10419            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
10420                if reason == "durable_command_sequence_invalid"
10421        ));
10422
10423        let descending = WorkflowState::new(
10424            vec![
10425                history_event(
10426                    "SideEffectRecorded",
10427                    json!({"sequence": 3, "result": result}),
10428                ),
10429                history_event(
10430                    "VersionMarkerRecorded",
10431                    json!({
10432                        "sequence": 2,
10433                        "change_id": "descending-marker",
10434                        "version": 1,
10435                        "min_supported": 1,
10436                        "max_supported": 1,
10437                    }),
10438                ),
10439            ],
10440            "rust-workers".to_string(),
10441            JSON_CODEC.to_string(),
10442            None,
10443        )
10444        .expect_err("new durable commands must remain strictly ordered");
10445        let Error::NonDeterministicReplay(failure) = descending else {
10446            panic!("expected typed replay failure");
10447        };
10448        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
10449        assert_eq!(failure.sequence, Some(2));
10450        assert_eq!(
10451            failure.expected.as_deref(),
10452            Some("workflow sequence greater than 3")
10453        );
10454        assert_eq!(failure.actual.as_deref(), Some("2"));
10455    }
10456
10457    #[test]
10458    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
10459        fn worker() -> Worker {
10460            let client = Client::new("http://127.0.0.1:8080").expect("client");
10461            let mut worker = Worker::new(client, "rust-workers");
10462            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
10463                ctx.wait_signal("finish").await?;
10464                let marker: String =
10465                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
10466                assert_eq!(marker, "after-finish");
10467                Ok(json!("finished"))
10468            });
10469            worker
10470        }
10471
10472        let marker =
10473            encode_value_envelope(&json!("after-finish"), JSON_CODEC).expect("side-effect result");
10474        let task = workflow_task(
10475            "rust.finish-after-gaps",
10476            vec![
10477                history_event(
10478                    "SignalWaitOpened",
10479                    json!({"sequence": 1, "signal_name": "finish"}),
10480                ),
10481                history_event(
10482                    "SignalReceived",
10483                    json!({
10484                        "signal_id": "increment-3",
10485                        "signal_name": "increment",
10486                        "workflow_sequence": 2,
10487                        "payload_codec": "json",
10488                        "arguments": {"codec": "json", "blob": "[3]"},
10489                    }),
10490                ),
10491                history_event(
10492                    "SignalReceived",
10493                    json!({
10494                        "signal_id": "increment-5",
10495                        "signal_name": "increment",
10496                        "workflow_sequence": 3,
10497                        "payload_codec": "json",
10498                        "arguments": {"codec": "json", "blob": "[5]"},
10499                    }),
10500                ),
10501                history_event(
10502                    "SignalReceived",
10503                    json!({
10504                        "signal_id": "finish",
10505                        "signal_name": "finish",
10506                        "workflow_sequence": 4,
10507                        "payload_codec": "json",
10508                        "arguments": {"codec": "json", "blob": "[]"},
10509                    }),
10510                ),
10511                history_event(
10512                    "SignalApplied",
10513                    json!({
10514                        "sequence": 1,
10515                        "signal_id": "finish",
10516                        "signal_name": "finish",
10517                        "payload_codec": "json",
10518                        "value": {"codec": "json", "blob": "[]"},
10519                    }),
10520                ),
10521                history_event(
10522                    "SideEffectRecorded",
10523                    json!({"sequence": 5, "result": marker}),
10524                ),
10525            ],
10526            JSON_CODEC,
10527        );
10528
10529        for _original_or_cold_worker in 0..2 {
10530            let commands = worker()
10531                .execute_workflow_task(task.clone())
10532                .expect("signal gaps preserve deterministic replay");
10533            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
10534            assert_eq!(commands[0]["type"], "complete_workflow");
10535            assert_eq!(
10536                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("workflow output"),
10537                json!("finished")
10538            );
10539        }
10540    }
10541
10542    #[test]
10543    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
10544        let ctx = workflow_context(Vec::new());
10545        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
10546        let mut task_context = TaskContext::from_waker(noop_waker_ref());
10547        assert!(matches!(
10548            sleep.as_mut().poll(&mut task_context),
10549            Poll::Ready(Err(Error::TimerDurationOverflow))
10550        ));
10551        assert!(ctx.take_commands().expect("commands").is_empty());
10552    }
10553
10554    #[test]
10555    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
10556        let client = Client::new("http://127.0.0.1:8080").expect("client");
10557        let mut worker = Worker::new(client, "rust-workers");
10558        worker.register_workflow("rust.timer", |ctx, _input| async move {
10559            ctx.sleep(Duration::from_secs(5)).await?;
10560            ctx.activity("after-timer", json!([])).await
10561        });
10562
10563        let task = |history_events| WorkflowTask {
10564            task_id: "wft-rust-timer-1".to_string(),
10565            workflow_id: Some("wf-rust-timer".to_string()),
10566            run_id: Some("run-rust-timer".to_string()),
10567            workflow_type: "rust.timer".to_string(),
10568            payload_codec: JSON_CODEC.to_string(),
10569            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10570            history_events,
10571            total_history_events: None,
10572            history_size_bytes: None,
10573            continue_as_new_recommended: None,
10574            history_budget_pressure: None,
10575            next_history_page_token: None,
10576            workflow_task_attempt: 1,
10577            workflow_signal_id: None,
10578            signal_name: None,
10579            signal_arguments: None,
10580            workflow_update_id: None,
10581            update_name: None,
10582            lease_owner: Some("rust-worker".to_string()),
10583        };
10584
10585        let initial = worker
10586            .execute_workflow_task(task(Vec::new()))
10587            .expect("initial timer task");
10588        assert_eq!(
10589            initial,
10590            vec![json!({"type": "start_timer", "delay_seconds": 5})]
10591        );
10592
10593        let replayed = worker
10594            .execute_workflow_task(task(vec![
10595                history_event(
10596                    "TimerScheduled",
10597                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10598                ),
10599                history_event(
10600                    "TimerFired",
10601                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10602                ),
10603                history_event(
10604                    "ActivityCompleted",
10605                    json!({
10606                        "sequence": 2,
10607                        "activity_type": "after-timer",
10608                        "payload_codec": "json",
10609                        "result": {"codec": "json", "blob": "\"done\""},
10610                    }),
10611                ),
10612            ]))
10613            .expect("replayed workflow task");
10614        assert_eq!(replayed.len(), 1);
10615        assert_eq!(replayed[0]["type"], "complete_workflow");
10616        assert_eq!(
10617            decode_wire_value(&replayed[0]["result"], JSON_CODEC).expect("result"),
10618            json!("done")
10619        );
10620    }
10621
10622    #[test]
10623    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
10624        let client = Client::new("http://127.0.0.1:8080").expect("client");
10625        let mut worker = Worker::new(client, "rust-workers");
10626        worker.register_workflow("rust.continue", |ctx, _input| async move {
10627            ctx.continue_as_new_with_options(
10628                ContinueAsNewOptions::new()
10629                    .workflow_type("rust.next")
10630                    .task_queue("next-workers"),
10631                json!([2, {"cursor": "next"}]),
10632            )
10633        });
10634
10635        let commands = worker
10636            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
10637            .expect("continue-as-new command");
10638
10639        assert_eq!(commands.len(), 1);
10640        assert_eq!(commands[0]["type"], "continue_as_new");
10641        assert_eq!(commands[0]["workflow_type"], "rust.next");
10642        assert_eq!(commands[0]["queue"], "next-workers");
10643        assert_eq!(
10644            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
10645                .expect("continue-as-new arguments"),
10646            json!([2, {"cursor": "next"}])
10647        );
10648    }
10649
10650    #[test]
10651    fn continue_as_new_preserves_typed_arguments() {
10652        let client = Client::new("http://127.0.0.1:8080").expect("client");
10653        let mut worker = Worker::new(client, "rust-workers");
10654        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
10655            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
10656            unreachable!("continue-as-new returns a control-flow error")
10657        });
10658
10659        let commands = worker
10660            .execute_workflow_task(workflow_task(
10661                "rust.typed-continue",
10662                Vec::new(),
10663                DEFAULT_CODEC,
10664            ))
10665            .expect("typed continue-as-new command");
10666
10667        assert_eq!(commands[0]["type"], "continue_as_new");
10668        assert_eq!(
10669            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
10670                .expect("typed continue arguments"),
10671            AvroValue::Array(vec![typed_fidelity_probe()])
10672        );
10673    }
10674
10675    #[test]
10676    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
10677        let client = Client::new("http://127.0.0.1:8080").expect("client");
10678        let mut worker = Worker::new(client, "rust-workers");
10679        worker.register_workflow("rust.continue", |ctx, _input| async move {
10680            ctx.continue_as_new(json!([2]))
10681        });
10682        let task = workflow_task(
10683            "rust.continue",
10684            vec![history_event(
10685                "WorkflowContinuedAsNew",
10686                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
10687            )],
10688            JSON_CODEC,
10689        );
10690
10691        for _worker_restart_or_redelivery in 0..2 {
10692            let commands = worker
10693                .execute_workflow_task(task.clone())
10694                .expect("recorded transition replays");
10695            assert!(
10696                commands.is_empty(),
10697                "replay must not emit another successor"
10698            );
10699        }
10700    }
10701
10702    #[test]
10703    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
10704        let ctx = workflow_context(Vec::new());
10705        let error = ctx
10706            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
10707            .expect_err("blank queue must be rejected");
10708
10709        let Error::InvalidContinueAsNewOptions(error) = error else {
10710            panic!("expected typed continue-as-new validation error");
10711        };
10712        assert_eq!(error.field, "task_queue");
10713        assert!(ctx.take_commands().expect("commands").is_empty());
10714    }
10715
10716    #[test]
10717    fn workflow_context_exposes_server_history_budget() {
10718        let client = Client::new("http://127.0.0.1:8080").expect("client");
10719        let mut worker = Worker::new(client, "rust-workers");
10720        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
10721            let budget = ctx.history_budget()?;
10722            Ok(json!({
10723                "events": budget.event_count,
10724                "bytes": budget.size_bytes,
10725                "recommended": budget.continue_as_new_recommended,
10726                "pressure": budget.pressure,
10727            }))
10728        });
10729        let task: WorkflowTask = serde_json::from_value(json!({
10730            "task_id": "task-history-budget",
10731            "workflow_type": "rust.history-budget",
10732            "payload_codec": JSON_CODEC,
10733            "history_events": [],
10734            "total_history_events": 480,
10735            "history_size_bytes": 1_048_576,
10736            "continue_as_new_recommended": true,
10737            "history_budget_pressure": "continue_as_new_recommended",
10738        }))
10739        .expect("published workflow task");
10740
10741        let commands = worker
10742            .execute_workflow_task(task)
10743            .expect("history-budget workflow");
10744        let result = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("result");
10745        assert_eq!(result["events"], 480);
10746        assert_eq!(result["bytes"], 1_048_576);
10747        assert_eq!(result["recommended"], true);
10748        assert_eq!(result["pressure"], "continue_as_new_recommended");
10749    }
10750
10751    #[test]
10752    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
10753        let client = Client::new("http://127.0.0.1:8080").expect("client");
10754        let mut worker = Worker::new(client, "rust-workers");
10755        worker.register_workflow("rust.failing", |_ctx, _input| async move {
10756            Err(Error::Codec("rust_conformance_failure".to_string()))
10757        });
10758        let task = WorkflowTask {
10759            task_id: "wft-rust-failing-1".to_string(),
10760            workflow_id: Some("wf-rust-failing".to_string()),
10761            run_id: Some("run-rust-failing".to_string()),
10762            workflow_type: "rust.failing".to_string(),
10763            payload_codec: JSON_CODEC.to_string(),
10764            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
10765            history_events: Vec::new(),
10766            total_history_events: Some(0),
10767            history_size_bytes: None,
10768            continue_as_new_recommended: None,
10769            history_budget_pressure: None,
10770            next_history_page_token: None,
10771            workflow_task_attempt: 1,
10772            workflow_signal_id: None,
10773            signal_name: None,
10774            signal_arguments: None,
10775            workflow_update_id: None,
10776            update_name: None,
10777            lease_owner: Some("rust-worker".to_string()),
10778        };
10779
10780        let commands = worker
10781            .execute_workflow_task(task)
10782            .expect("handler failure becomes a workflow command");
10783
10784        assert_eq!(commands.len(), 1);
10785        assert_eq!(commands[0]["type"], "fail_workflow");
10786        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
10787        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
10788        assert_eq!(commands[0]["non_retryable"], false);
10789        assert_eq!(
10790            commands[0]["message"],
10791            "codec error: rust_conformance_failure"
10792        );
10793        assert_eq!(
10794            commands[0]["exception"]["message"],
10795            "codec error: rust_conformance_failure"
10796        );
10797    }
10798
10799    #[test]
10800    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
10801        let client = Client::new("http://127.0.0.1:8080").expect("client");
10802        let mut worker = Worker::new(client, "rust-workers");
10803        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
10804            let _: String = ctx.side_effect(|| "captured".to_string())?;
10805            Err(Error::WorkerLoop("application failure".to_string()))
10806        });
10807
10808        let commands = worker
10809            .execute_workflow_task(workflow_task(
10810                "rust.failing-after-side-effect",
10811                Vec::new(),
10812                JSON_CODEC,
10813            ))
10814            .expect("ordinary failure remains a workflow decision");
10815
10816        assert_eq!(commands.len(), 2);
10817        assert_eq!(commands[0]["type"], "record_side_effect");
10818        assert_eq!(commands[1]["type"], "fail_workflow");
10819    }
10820
10821    #[test]
10822    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
10823        let client = Client::new("http://127.0.0.1:8080").expect("client");
10824        let mut worker = Worker::new(client, "rust-workers");
10825        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
10826            Err(Error::WorkerLoop("application failure".to_string()))
10827        });
10828        let result =
10829            encode_value_envelope(&json!("committed"), JSON_CODEC).expect("side-effect result");
10830
10831        let error = worker
10832            .execute_workflow_task(workflow_task(
10833                "rust.removed-side-effect",
10834                vec![history_event(
10835                    "SideEffectRecorded",
10836                    json!({"sequence": 1, "result": result}),
10837                )],
10838                JSON_CODEC,
10839            ))
10840            .expect_err("removed committed history must not become fail_workflow");
10841
10842        let Error::NonDeterministicReplay(failure) = error else {
10843            panic!("expected typed replay failure");
10844        };
10845        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10846        assert_eq!(failure.sequence, Some(1));
10847        assert_eq!(failure.expected.as_deref(), Some("side effect"));
10848    }
10849
10850    #[test]
10851    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
10852        let client = Client::new("http://127.0.0.1:8080").expect("client");
10853        let mut worker = Worker::new(client, "rust-workers");
10854        worker.register_workflow(
10855            "rust.side-effect-before-marker-error",
10856            |ctx, _input| async move {
10857                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
10858                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
10859                ctx.get_version("restart-safe", 2, 2)?;
10860                Ok(Value::Null)
10861            },
10862        );
10863
10864        let error = worker
10865            .execute_workflow_task(workflow_task(
10866                "rust.side-effect-before-marker-error",
10867                vec![history_event(
10868                    "VersionMarkerRecorded",
10869                    json!({
10870                        "sequence": 1,
10871                        "change_id": "restart-safe",
10872                        "version": 1,
10873                        "min_supported": 1,
10874                        "max_supported": 1,
10875                    }),
10876                )],
10877                JSON_CODEC,
10878            ))
10879            .expect_err("replay error must return no queued workflow commands");
10880
10881        let Error::NonDeterministicReplay(failure) = error else {
10882            panic!("expected typed replay failure");
10883        };
10884        assert_eq!(failure.reason, "version_marker_incompatible_range");
10885        assert_eq!(failure.sequence, Some(1));
10886    }
10887
10888    #[test]
10889    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
10890        let client = Client::new("http://127.0.0.1:8080").expect("client");
10891        let mut worker = Worker::new(client, "rust-workers");
10892        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
10893            ctx.sleep(Duration::from_secs(5)).await?;
10894            Ok(json!({"status": "timer fired"}))
10895        });
10896
10897        let task = WorkflowTask {
10898            task_id: "wft-rust-timer-pending".to_string(),
10899            workflow_id: Some("wf-rust-timer".to_string()),
10900            run_id: Some("run-rust-timer".to_string()),
10901            workflow_type: "rust.timer.pending".to_string(),
10902            payload_codec: JSON_CODEC.to_string(),
10903            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10904            history_events: vec![history_event(
10905                "TimerScheduled",
10906                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10907            )],
10908            total_history_events: Some(1),
10909            history_size_bytes: None,
10910            continue_as_new_recommended: None,
10911            history_budget_pressure: None,
10912            next_history_page_token: None,
10913            workflow_task_attempt: 1,
10914            workflow_signal_id: None,
10915            signal_name: None,
10916            signal_arguments: None,
10917            workflow_update_id: None,
10918            update_name: None,
10919            lease_owner: Some("rust-worker".to_string()),
10920        };
10921
10922        for _redelivery_or_restart in 0..2 {
10923            let commands = worker
10924                .execute_workflow_task(task.clone())
10925                .expect("recorded timer remains pending");
10926            assert!(
10927                commands.is_empty(),
10928                "recorded timer must not be rescheduled"
10929            );
10930        }
10931    }
10932
10933    #[test]
10934    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
10935        let client = Client::new("http://127.0.0.1:8080").expect("client");
10936        let mut worker = Worker::new(client, "rust-workers");
10937        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
10938            Ok(json!({"status": "completed"}))
10939        });
10940        let task = WorkflowTask {
10941            task_id: "wft-rust-timer-removed".to_string(),
10942            workflow_id: Some("wf-rust-timer".to_string()),
10943            run_id: Some("run-rust-timer".to_string()),
10944            workflow_type: "rust.timer.removed".to_string(),
10945            payload_codec: JSON_CODEC.to_string(),
10946            arguments: Some(json!({"codec": "json", "blob": "[]"})),
10947            history_events: vec![
10948                history_event(
10949                    "TimerScheduled",
10950                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10951                ),
10952                history_event(
10953                    "TimerFired",
10954                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
10955                ),
10956            ],
10957            total_history_events: Some(2),
10958            history_size_bytes: None,
10959            continue_as_new_recommended: None,
10960            history_budget_pressure: None,
10961            next_history_page_token: None,
10962            workflow_task_attempt: 1,
10963            workflow_signal_id: None,
10964            signal_name: None,
10965            signal_arguments: None,
10966            workflow_update_id: None,
10967            update_name: None,
10968            lease_owner: Some("rust-worker".to_string()),
10969        };
10970
10971        let Error::NonDeterministicReplay(failure) = worker
10972            .execute_workflow_task(task)
10973            .expect_err("removed timer must fail replay")
10974        else {
10975            panic!("expected typed replay failure");
10976        };
10977        assert_eq!(failure.reason, "recorded_commands_unconsumed");
10978        assert_eq!(failure.sequence, Some(1));
10979    }
10980
10981    #[test]
10982    fn workflow_context_emits_explicit_child_workflow_contract() {
10983        let ctx = WorkflowContext {
10984            state: Arc::new(Mutex::new(
10985                WorkflowState::new_with_identity(
10986                    Vec::new(),
10987                    Some("wf-parent".to_string()),
10988                    Some("run-parent".to_string()),
10989                    "parent-workers".to_string(),
10990                    JSON_CODEC.to_string(),
10991                    None,
10992                )
10993                .expect("workflow state"),
10994            )),
10995        };
10996        let options = ChildWorkflowOptions::new("python-workers")
10997            .parent_close_policy(ParentClosePolicy::RequestCancel)
10998            .retry_policy(ChildWorkflowRetryPolicy {
10999                max_attempts: Some(3),
11000                backoff_seconds: vec![1, 5],
11001                non_retryable_error_types: vec!["ValidationError".to_string()],
11002            })
11003            .execution_timeout_seconds(600)
11004            .run_timeout_seconds(120);
11005        let mut call = Box::pin(ctx.start_child_workflow(
11006            "python.fulfil-order",
11007            options,
11008            json!([{"order_id": "order-42"}]),
11009        ));
11010        let mut task_context = TaskContext::from_waker(noop_waker_ref());
11011
11012        assert!(matches!(
11013            call.as_mut().poll(&mut task_context),
11014            Poll::Pending
11015        ));
11016        let commands = ctx.take_commands().expect("commands");
11017        assert_eq!(commands.len(), 1);
11018        let command = &commands[0];
11019        assert_eq!(command["type"], "start_child_workflow");
11020        assert_eq!(command["workflow_type"], "python.fulfil-order");
11021        assert_eq!(command["queue"], "python-workers");
11022        assert_eq!(command["parent_close_policy"], "request_cancel");
11023        assert_eq!(command["retry_policy"]["max_attempts"], 3);
11024        assert_eq!(command["execution_timeout_seconds"], 600);
11025        assert_eq!(command["run_timeout_seconds"], 120);
11026        assert_eq!(
11027            decode_wire_value(&command["arguments"], JSON_CODEC).expect("child args"),
11028            json!([{"order_id": "order-42"}])
11029        );
11030    }
11031
11032    fn child_parent_worker() -> Worker {
11033        let client = Client::new("http://127.0.0.1:8080").expect("client");
11034        let mut worker = Worker::new(client, "rust-parent-workers");
11035        worker.register_workflow("rust.parent", |ctx, _input| async move {
11036            let child = ctx
11037                .start_child_workflow(
11038                    "python.child",
11039                    ChildWorkflowOptions::new("python-child-workers")
11040                        .parent_close_policy(ParentClosePolicy::Terminate),
11041                    json!([{"codec_probe": [1, true, "rust"]}]),
11042                )
11043                .await?;
11044            Ok(json!({
11045                "parent_workflow_id": child.parent.workflow_id,
11046                "parent_run_id": child.parent.run_id,
11047                "child_workflow_id": child.child.workflow_id,
11048                "child_run_id": child.child.run_id,
11049                "child_workflow_type": child.child_workflow_type,
11050                "result": child.result,
11051            }))
11052        });
11053        worker
11054    }
11055
11056    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
11057        WorkflowTask {
11058            task_id: "wft-child-parent".to_string(),
11059            workflow_id: Some("wf-parent".to_string()),
11060            run_id: Some("run-parent".to_string()),
11061            workflow_type: "rust.parent".to_string(),
11062            payload_codec: JSON_CODEC.to_string(),
11063            arguments: Some(encode_value_envelope(&json!([]), JSON_CODEC).expect("input")),
11064            history_events: vec![
11065                HistoryEvent {
11066                    event_type: "ChildWorkflowScheduled".to_string(),
11067                    payload: json!({
11068                        "sequence": 1,
11069                        "child_call_id": "call-child",
11070                        "child_workflow_instance_id": "wf-child",
11071                        "child_workflow_run_id": "run-child",
11072                        "child_workflow_type": "python.child",
11073                    }),
11074                    raw: HashMap::new(),
11075                },
11076                HistoryEvent {
11077                    event_type: event_type.to_string(),
11078                    payload,
11079                    raw: HashMap::new(),
11080                },
11081            ],
11082            total_history_events: Some(2),
11083            history_size_bytes: None,
11084            continue_as_new_recommended: None,
11085            history_budget_pressure: None,
11086            next_history_page_token: None,
11087            workflow_task_attempt: 1,
11088            workflow_signal_id: None,
11089            signal_name: None,
11090            signal_arguments: None,
11091            workflow_update_id: None,
11092            update_name: None,
11093            lease_owner: Some("rust-worker".to_string()),
11094        }
11095    }
11096
11097    #[test]
11098    fn committed_child_result_replays_without_starting_a_duplicate() {
11099        let worker = child_parent_worker();
11100        let task = child_parent_task(
11101            "ChildRunCompleted",
11102            json!({
11103                "sequence": 1,
11104                "child_call_id": "call-child",
11105                "child_workflow_instance_id": "wf-child",
11106                "child_workflow_run_id": "run-child",
11107                "child_workflow_type": "python.child",
11108                "payload_codec": "json",
11109                "result": {"codec": "json", "blob": "{\"from\":\"python\",\"ok\":true}"},
11110            }),
11111        );
11112
11113        for _restart in 0..2 {
11114            let commands = worker
11115                .execute_workflow_task(task.clone())
11116                .expect("replayed parent task");
11117            assert_eq!(commands.len(), 1);
11118            assert_eq!(commands[0]["type"], "complete_workflow");
11119            assert!(!commands
11120                .iter()
11121                .any(|command| command["type"] == "start_child_workflow"));
11122            let output =
11123                decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11124            assert_eq!(output["parent_workflow_id"], "wf-parent");
11125            assert_eq!(output["parent_run_id"], "run-parent");
11126            assert_eq!(output["child_workflow_id"], "wf-child");
11127            assert_eq!(output["child_run_id"], "run-child");
11128            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
11129        }
11130    }
11131
11132    #[test]
11133    fn typed_child_arguments_and_results_survive_replay() {
11134        let client = Client::new("http://127.0.0.1:8080").expect("client");
11135        let mut worker = Worker::new(client, "rust-parent-workers");
11136        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
11137            let child = ctx
11138                .start_child_workflow_avro_value(
11139                    "python.typed-child",
11140                    ChildWorkflowOptions::new("python-workers"),
11141                    AvroValue::Array(vec![typed_fidelity_probe()]),
11142                )
11143                .await?;
11144            Ok(child.result)
11145        });
11146
11147        let initial = worker
11148            .execute_workflow_task(workflow_task(
11149                "rust.typed-parent",
11150                Vec::new(),
11151                DEFAULT_CODEC,
11152            ))
11153            .expect("typed child start");
11154        assert_eq!(initial[0]["type"], "start_child_workflow");
11155        assert_eq!(
11156            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
11157                .expect("typed child arguments"),
11158            AvroValue::Array(vec![typed_fidelity_probe()])
11159        );
11160
11161        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
11162            .expect("typed child result");
11163        let task = workflow_task(
11164            "rust.typed-parent",
11165            vec![
11166                history_event(
11167                    "ChildWorkflowScheduled",
11168                    json!({
11169                        "sequence": 1,
11170                        "child_call_id": "call-typed",
11171                        "child_workflow_instance_id": "wf-child",
11172                        "child_workflow_run_id": "run-child",
11173                        "child_workflow_type": "python.typed-child",
11174                    }),
11175                ),
11176                history_event(
11177                    "ChildRunCompleted",
11178                    json!({
11179                        "sequence": 1,
11180                        "child_call_id": "call-typed",
11181                        "child_workflow_instance_id": "wf-child",
11182                        "child_workflow_run_id": "run-child",
11183                        "child_workflow_type": "python.typed-child",
11184                        "payload_codec": DEFAULT_CODEC,
11185                        "result": result,
11186                    }),
11187                ),
11188            ],
11189            DEFAULT_CODEC,
11190        );
11191
11192        let commands = worker
11193            .execute_workflow_task(task)
11194            .expect("typed child replay");
11195        assert_eq!(commands[0]["type"], "complete_workflow");
11196        assert_eq!(
11197            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
11198                .expect("typed parent result"),
11199            typed_fidelity_probe()
11200        );
11201    }
11202
11203    #[test]
11204    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
11205        let worker = child_parent_worker();
11206        let mut task = child_parent_task("unused", Value::Null);
11207        task.history_events.truncate(1);
11208        task.total_history_events = Some(1);
11209
11210        for _redelivery_or_restart in 0..2 {
11211            let commands = worker
11212                .execute_workflow_task(task.clone())
11213                .expect("recorded child remains pending");
11214            assert!(
11215                commands.is_empty(),
11216                "recorded pending child must not be started again"
11217            );
11218        }
11219    }
11220
11221    #[test]
11222    fn child_cancellation_becomes_stable_parent_failure_command() {
11223        let worker = child_parent_worker();
11224        let task = child_parent_task(
11225            "ChildRunCancelled",
11226            json!({
11227                "sequence": 1,
11228                "child_workflow_instance_id": "wf-child",
11229                "child_workflow_run_id": "run-child",
11230                "child_workflow_type": "python.child",
11231                "failure_id": "failure-child",
11232                "failure_category": "cancelled",
11233                "message": "cancelled by parent-close policy",
11234            }),
11235        );
11236
11237        let commands = worker
11238            .execute_workflow_task(task)
11239            .expect("parent settlement");
11240        assert_eq!(commands.len(), 1);
11241        assert_eq!(commands[0]["type"], "fail_workflow");
11242        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
11243        assert_eq!(
11244            commands[0]["exception"]["properties"]["reason"],
11245            "cancelled"
11246        );
11247        assert_eq!(
11248            commands[0]["exception"]["properties"]["child_workflow_run_id"],
11249            "run-child"
11250        );
11251    }
11252
11253    #[test]
11254    fn workflow_can_handle_typed_child_failure() {
11255        let client = Client::new("http://127.0.0.1:8080").expect("client");
11256        let mut worker = Worker::new(client, "rust-parent-workers");
11257        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
11258            match ctx
11259                .start_child_workflow(
11260                    "python.child",
11261                    ChildWorkflowOptions::new("python-child-workers"),
11262                    json!([]),
11263                )
11264                .await
11265            {
11266                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
11267                    "reason": failure.reason,
11268                    "failure_id": failure.failure_id,
11269                    "exception_class": failure.exception_class,
11270                    "child_run_id": failure.child_workflow_run_id,
11271                })),
11272                Err(error) => Err(error),
11273                Ok(_) => Err(Error::WorkerLoop(
11274                    "child unexpectedly succeeded".to_string(),
11275                )),
11276            }
11277        });
11278        let mut task = child_parent_task(
11279            "ChildRunFailed",
11280            json!({
11281                "sequence": 1,
11282                "child_workflow_instance_id": "wf-child",
11283                "child_workflow_run_id": "run-child",
11284                "child_workflow_type": "python.child",
11285                "failure_id": "failure-child",
11286                "failure_category": "child_workflow",
11287                "message": "payment rejected",
11288                "exception": {
11289                    "type": "PaymentRejected",
11290                    "class": "payments.PaymentRejected",
11291                    "message": "payment rejected"
11292                }
11293            }),
11294        );
11295        task.workflow_type = "rust.handled-parent".to_string();
11296
11297        let commands = worker.execute_workflow_task(task).expect("handled failure");
11298        assert_eq!(commands[0]["type"], "complete_workflow");
11299        let output = decode_wire_value(&commands[0]["result"], JSON_CODEC).expect("parent output");
11300        assert_eq!(output["reason"], "child_workflow");
11301        assert_eq!(output["failure_id"], "failure-child");
11302        assert_eq!(output["exception_class"], "payments.PaymentRejected");
11303        assert_eq!(output["child_run_id"], "run-child");
11304    }
11305
11306    #[test]
11307    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
11308        let client = Client::new("http://127.0.0.1:8080").expect("client");
11309        let mut worker = Worker::new(client, "rust-workers");
11310
11311        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
11312            let signal = ctx.wait_signal("start").await?;
11313            let name = signal
11314                .first()
11315                .and_then(|value| value.as_str())
11316                .unwrap_or("world");
11317            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
11318            Ok(json!({
11319                "greeting": greeting,
11320                "language": "rust"
11321            }))
11322        });
11323
11324        let signal_arguments =
11325            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
11326        let task = WorkflowTask {
11327            task_id: "wft-rust-signal-1".to_string(),
11328            workflow_id: Some("wf-rust-hello".to_string()),
11329            run_id: Some("run-rust-hello".to_string()),
11330            workflow_type: "rust.hello_workflow".to_string(),
11331            payload_codec: DEFAULT_CODEC.to_string(),
11332            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11333            history_events: vec![HistoryEvent {
11334                event_type: "SignalReceived".to_string(),
11335                payload: json!({
11336                    "signal_id": "sig-rust-1",
11337                    "signal_name": "start"
11338                }),
11339                raw: HashMap::new(),
11340            }],
11341            total_history_events: Some(1),
11342            history_size_bytes: None,
11343            continue_as_new_recommended: None,
11344            history_budget_pressure: None,
11345            next_history_page_token: None,
11346            workflow_task_attempt: 1,
11347            workflow_signal_id: Some("sig-rust-1".to_string()),
11348            signal_name: Some("start".to_string()),
11349            signal_arguments: Some(signal_arguments),
11350            workflow_update_id: None,
11351            update_name: None,
11352            lease_owner: Some("rust-worker".to_string()),
11353        };
11354
11355        let commands = worker.execute_workflow_task(task).expect("workflow task");
11356
11357        assert_eq!(commands.len(), 1);
11358        assert_eq!(commands[0]["type"], "schedule_activity");
11359        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
11360        assert_eq!(
11361            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
11362            json!(["Rust"])
11363        );
11364    }
11365
11366    #[test]
11367    fn workflow_task_appends_paginated_history_events() {
11368        let mut task = WorkflowTask {
11369            task_id: "wft-rust-pages-1".to_string(),
11370            workflow_id: Some("wf-rust-pages".to_string()),
11371            run_id: Some("run-rust-pages".to_string()),
11372            workflow_type: "rust.hello_workflow".to_string(),
11373            payload_codec: DEFAULT_CODEC.to_string(),
11374            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
11375            history_events: vec![HistoryEvent {
11376                event_type: "WorkflowStarted".to_string(),
11377                payload: json!({}),
11378                raw: HashMap::new(),
11379            }],
11380            total_history_events: Some(3),
11381            history_size_bytes: None,
11382            continue_as_new_recommended: None,
11383            history_budget_pressure: None,
11384            next_history_page_token: Some("MQ==".to_string()),
11385            workflow_task_attempt: 1,
11386            workflow_signal_id: None,
11387            signal_name: None,
11388            signal_arguments: None,
11389            workflow_update_id: None,
11390            update_name: None,
11391            lease_owner: Some("rust-worker".to_string()),
11392        };
11393
11394        task.append_history_page(WorkflowTaskHistoryPage {
11395            history_events: vec![
11396                HistoryEvent {
11397                    event_type: "SignalReceived".to_string(),
11398                    payload: json!({
11399                        "signal_id": "sig-rust-1",
11400                        "signal_name": "start",
11401                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
11402                            .expect("signal arguments")
11403                    }),
11404                    raw: HashMap::new(),
11405                },
11406                HistoryEvent {
11407                    event_type: "MarkerRecorded".to_string(),
11408                    payload: json!({"sequence": 3}),
11409                    raw: HashMap::new(),
11410                },
11411            ],
11412            total_history_events: Some(3),
11413            next_history_page_token: None,
11414        });
11415
11416        assert_eq!(task.history_events.len(), 3);
11417        assert_eq!(task.total_history_events, Some(3));
11418        assert_eq!(task.next_history_page_token, None);
11419
11420        let signal = task
11421            .history_events
11422            .iter()
11423            .find(|event| event.event_type == "SignalReceived")
11424            .expect("signal event");
11425        assert_eq!(
11426            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
11427            vec![AvroValue::String("Rust".to_string())]
11428        );
11429    }
11430
11431    #[tokio::test]
11432    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
11433        let client = Client::new("http://127.0.0.1:8080").expect("client");
11434        let mut worker = Worker::new(client, "rust-workers");
11435        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11436        worker.register_query("counter", "current", |ctx, _args| async move {
11437            let mut count = 0_i64;
11438            for signal in ctx.signal_events() {
11439                let value = signal
11440                    .arguments
11441                    .first()
11442                    .and_then(Value::as_i64)
11443                    .unwrap_or_default();
11444                match signal.name.as_str() {
11445                    "increment" => count += value,
11446                    "set" => count = value,
11447                    _ => {}
11448                }
11449            }
11450            Ok(json!(count))
11451        });
11452
11453        let task = QueryTask {
11454            query_task_id: "query-rust-counter".to_string(),
11455            query_task_attempt: 1,
11456            lease_owner: Some("rust-worker".to_string()),
11457            workflow_id: Some("counter-1".to_string()),
11458            run_id: Some("run-counter-1".to_string()),
11459            workflow_type: "counter".to_string(),
11460            query_name: "current".to_string(),
11461            payload_codec: DEFAULT_CODEC.to_string(),
11462            workflow_arguments: Some(
11463                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
11464            ),
11465            query_arguments: Some(
11466                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
11467            ),
11468            history_events: vec![
11469                HistoryEvent {
11470                    event_type: "SignalReceived".to_string(),
11471                    payload: json!({
11472                        "signal_id": "php-signal-1",
11473                        "signal_name": "increment",
11474                        "workflow_sequence": 1,
11475                        "payload_codec": DEFAULT_CODEC,
11476                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
11477                    }),
11478                    raw: HashMap::new(),
11479                },
11480                HistoryEvent {
11481                    event_type: "SignalReceived".to_string(),
11482                    payload: json!({
11483                        "signal_id": "python-signal-2",
11484                        "signal_name": "increment",
11485                        "workflow_sequence": 2,
11486                        "payload_codec": JSON_CODEC,
11487                        "arguments": encode_value_envelope(&json!([5]), JSON_CODEC).expect("python json signal")
11488                    }),
11489                    raw: HashMap::new(),
11490                },
11491                HistoryEvent {
11492                    event_type: "SignalReceived".to_string(),
11493                    payload: json!({
11494                        "signal_id": "rust-signal-3",
11495                        "signal_name": "set",
11496                        "workflow_sequence": 3,
11497                        "payload_codec": DEFAULT_CODEC,
11498                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
11499                    }),
11500                    raw: HashMap::new(),
11501                },
11502            ],
11503            history_export: None,
11504            run_status: Some("completed".to_string()),
11505        };
11506
11507        let result = worker.execute_query_task(task).await.expect("query result");
11508        assert_eq!(result.into_json().expect("query projection"), json!(0));
11509    }
11510
11511    #[tokio::test]
11512    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
11513        let worker = replay_counter_worker();
11514        let running_history = json!([
11515            {
11516                "type": "ActivityCompleted",
11517                "payload": {
11518                    "sequence": 1,
11519                    "activity_type": "load-counter",
11520                    "payload_codec": "json",
11521                    "result": {"codec": "json", "blob": "\"loaded\""}
11522                }
11523            },
11524            {
11525                "type": "SignalWaitOpened",
11526                "payload": {
11527                    "sequence": 3,
11528                    "signal_name": "increment"
11529                }
11530            },
11531            {
11532                "type": "SignalReceived",
11533                "payload": {
11534                    "signal_id": "signal-3",
11535                    "signal_name": "increment",
11536                    "workflow_sequence": 2,
11537                    "payload_codec": "json",
11538                    "arguments": {"codec": "json", "blob": "[3]"}
11539                }
11540            },
11541            {
11542                "type": "SignalApplied",
11543                "payload": {
11544                    "sequence": 3,
11545                    "signal_id": "signal-3",
11546                    "signal_name": "increment",
11547                    "payload_codec": "json",
11548                    "value": {"codec": "json", "blob": "[3]"}
11549                }
11550            }
11551        ]);
11552
11553        let running = worker
11554            .execute_query_task(replay_counter_query(
11555                "current",
11556                running_history.clone(),
11557                "running",
11558            ))
11559            .await
11560            .expect("running replay query");
11561        assert_eq!(
11562            running.clone().into_json().expect("query projection"),
11563            json!({"loaded": "loaded", "count": 3, "finished": false})
11564        );
11565
11566        let detached = worker
11567            .execute_query_task(replay_counter_query(
11568                "detached-mutation",
11569                running_history.clone(),
11570                "running",
11571            ))
11572            .await
11573            .expect("query mutates only its detached state clone");
11574        assert_eq!(detached.into_json().expect("query projection"), json!(999));
11575        let failed = worker
11576            .execute_query_task(replay_counter_query(
11577                "failed-mutation",
11578                running_history.clone(),
11579                "running",
11580            ))
11581            .await
11582            .expect_err("failed query");
11583        assert_eq!(failed.reason, "query_rejected");
11584        let unchanged = worker
11585            .execute_query_task(replay_counter_query("current", running_history, "running"))
11586            .await
11587            .expect("later query reconstructs unchanged state");
11588        assert_eq!(unchanged, running);
11589
11590        let restarted_worker = replay_counter_worker();
11591        let restarted_task: QueryTask = serde_json::from_value(json!({
11592            "query_task_id": "query-after-restart",
11593            "workflow_id": "counter-1",
11594            "run_id": "run-counter-1",
11595            "workflow_type": "replay-counter",
11596            "query_name": "current",
11597            "payload_codec": "json",
11598            "workflow_arguments": {"codec": "json", "blob": "[]"},
11599            "query_arguments": {"codec": "json", "blob": "[]"},
11600            "history_events": [],
11601            "history_export": {
11602                "payloads": {"codec": "json"},
11603                "history_events": [
11604                    {
11605                        "type": "ActivityCompleted",
11606                        "payload": {
11607                            "sequence": 1,
11608                            "activity_type": "load-counter",
11609                            "payload_codec": "json",
11610                            "result": null
11611                        }
11612                    },
11613                    {
11614                        "type": "SignalWaitOpened",
11615                        "payload": {
11616                            "sequence": 3,
11617                            "signal_name": "increment"
11618                        }
11619                    },
11620                    {
11621                        "type": "SignalReceived",
11622                        "payload": {
11623                            "signal_id": "signal-3",
11624                            "signal_name": "increment",
11625                            "workflow_sequence": 2
11626                        }
11627                    },
11628                    {
11629                        "type": "SignalApplied",
11630                        "payload": {
11631                            "sequence": 3,
11632                            "signal_id": "signal-3",
11633                            "signal_name": "increment"
11634                        }
11635                    },
11636                    {
11637                        "type": "SignalWaitOpened",
11638                        "payload": {
11639                            "sequence": 5,
11640                            "signal_name": "increment"
11641                        }
11642                    },
11643                    {
11644                        "type": "SignalReceived",
11645                        "payload": {
11646                            "signal_id": "signal-5",
11647                            "signal_name": "increment",
11648                            "workflow_sequence": 4
11649                        }
11650                    },
11651                    {
11652                        "type": "SignalApplied",
11653                        "payload": {
11654                            "sequence": 5,
11655                            "signal_id": "signal-5",
11656                            "signal_name": "increment"
11657                        }
11658                    }
11659                ],
11660                "activities": [{
11661                    "sequence": 1,
11662                    "activity_type": "load-counter",
11663                    "payload_codec": "json",
11664                    "result": {"codec": "json", "blob": "\"loaded\""}
11665                }],
11666                "signals": [
11667                    {
11668                        "id": "signal-3",
11669                        "name": "increment",
11670                        "workflow_sequence": 2,
11671                        "payload_codec": "json",
11672                        "arguments": "[3]"
11673                    },
11674                    {
11675                        "id": "signal-5",
11676                        "name": "increment",
11677                        "workflow_sequence": 4,
11678                        "payload_codec": "json",
11679                        "arguments": "[5]"
11680                    }
11681                ]
11682            },
11683            "run_status": "completed"
11684        }))
11685        .expect("cold replay query task");
11686        let completed = restarted_worker
11687            .execute_query_task(restarted_task)
11688            .await
11689            .expect("completed cold replay query");
11690        assert_eq!(
11691            completed.into_json().expect("query projection"),
11692            json!({"loaded": "loaded", "count": 8, "finished": true})
11693        );
11694    }
11695
11696    #[tokio::test]
11697    async fn replayed_query_replay_failures_are_machine_readable() {
11698        let worker = replay_counter_worker();
11699        let task = replay_counter_query(
11700            "current",
11701            json!([{
11702                "type": "ActivityCompleted",
11703                "payload": {
11704                    "sequence": 1,
11705                    "payload_codec": "json",
11706                    "result": {"codec": "json", "blob": "{"}
11707                }
11708            }]),
11709            "running",
11710        );
11711        let failure = worker
11712            .execute_query_task(task)
11713            .await
11714            .expect_err("invalid replay history payload");
11715        assert_eq!(failure.reason, "query_workflow_state_unavailable");
11716        assert_eq!(failure.failure_type, "QueryWorkflowStateUnavailable");
11717    }
11718
11719    #[tokio::test]
11720    async fn query_task_restores_compact_history_from_export() {
11721        let client = Client::new("http://127.0.0.1:8080").expect("client");
11722        let mut worker = Worker::new(client, "rust-workers");
11723        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11724        worker.register_query("counter", "current", |ctx, _args| async move {
11725            Ok(json!(ctx.signals("increment")[0][0]))
11726        });
11727        let task: QueryTask = serde_json::from_value(json!({
11728            "query_task_id": "query-export",
11729            "workflow_type": "counter",
11730            "query_name": "current",
11731            "payload_codec": "json",
11732            "workflow_arguments": {"codec": "json", "blob": "[]"},
11733            "query_arguments": {"codec": "json", "blob": "[]"},
11734            "history_events": [],
11735            "history_export": {
11736                "payloads": {"codec": "json"},
11737                "history_events": [{
11738                    "type": "SignalReceived",
11739                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
11740                }],
11741                "signals": [{
11742                    "id": "signal-export",
11743                    "name": "increment",
11744                    "status": "applied",
11745                    "workflow_sequence": 1,
11746                    "payload_codec": "json",
11747                    "arguments": "[9]"
11748                }]
11749            }
11750        }))
11751        .expect("query task");
11752
11753        let result = worker.execute_query_task(task).await.expect("query result");
11754        assert_eq!(result.into_json().expect("query projection"), json!(9));
11755    }
11756
11757    #[tokio::test]
11758    async fn query_task_failures_have_stable_reasons() {
11759        let client = Client::new("http://127.0.0.1:8080").expect("client");
11760        let mut worker = Worker::new(client, "rust-workers");
11761        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11762        worker.register_query(
11763            "counter",
11764            "current",
11765            |_ctx, _args| async move { Ok(json!(0)) },
11766        );
11767
11768        let base_task = QueryTask {
11769            query_task_id: "query-errors".to_string(),
11770            query_task_attempt: 1,
11771            lease_owner: None,
11772            workflow_id: Some("counter-errors".to_string()),
11773            run_id: Some("run-errors".to_string()),
11774            workflow_type: "counter".to_string(),
11775            query_name: "missing".to_string(),
11776            payload_codec: JSON_CODEC.to_string(),
11777            workflow_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11778            query_arguments: Some(json!({"codec": "json", "blob": "[]"})),
11779            history_events: Vec::new(),
11780            history_export: None,
11781            run_status: Some("running".to_string()),
11782        };
11783
11784        let unknown = worker
11785            .execute_query_task(base_task.clone())
11786            .await
11787            .expect_err("unknown query");
11788        assert_eq!(unknown.reason, "rejected_unknown_query");
11789
11790        let mut malformed = base_task;
11791        malformed.query_name = "current".to_string();
11792        malformed.query_arguments = Some(json!({"codec": "json", "blob": "{"}));
11793        let malformed = worker
11794            .execute_query_task(malformed)
11795            .await
11796            .expect_err("malformed payload");
11797        assert_eq!(malformed.reason, "query_payload_decode_failed");
11798
11799        let client = Client::new("http://127.0.0.1:8080").expect("client");
11800        let mut unavailable_worker = Worker::new(client, "rust-workers");
11801        unavailable_worker
11802            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
11803        let unavailable_task: QueryTask = serde_json::from_value(json!({
11804            "query_task_id": "query-unavailable",
11805            "workflow_type": "counter",
11806            "query_name": "current",
11807            "payload_codec": "json",
11808            "workflow_arguments": {"codec": "json", "blob": "[]"},
11809            "query_arguments": {"codec": "json", "blob": "[]"}
11810        }))
11811        .expect("query task");
11812        let unavailable = unavailable_worker
11813            .execute_query_task(unavailable_task)
11814            .await
11815            .expect_err("query handler unavailable");
11816        assert_eq!(unavailable.reason, "query_handler_unavailable");
11817    }
11818
11819    #[tokio::test]
11820    async fn client_query_decodes_result_and_typed_failure() {
11821        let server = MockWorkerServer::start();
11822        let client = Client::builder(server.base_url())
11823            .timeout(Duration::from_secs(2))
11824            .build()
11825            .expect("client");
11826
11827        let result = client
11828            .query_workflow("counter-1", "current", json!([]))
11829            .await
11830            .expect("query result");
11831        assert_eq!(result, json!({"count": 8}));
11832
11833        let error = client
11834            .query_workflow("counter-1", "missing", json!([]))
11835            .await
11836            .expect_err("unknown query");
11837        let Error::QueryFailed(failure) = error else {
11838            panic!("expected typed query failure");
11839        };
11840        assert_eq!(failure.status, 404);
11841        assert_eq!(failure.reason, "rejected_unknown_query");
11842    }
11843
11844    #[tokio::test]
11845    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
11846        let server = MockWorkerServer::start();
11847        let client = Client::builder(server.base_url())
11848            .timeout(Duration::from_secs(2))
11849            .build()
11850            .expect("client");
11851        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
11852
11853        client
11854            .start_workflow(
11855                "typed.echo",
11856                "rust-workers",
11857                "typed-start",
11858                arguments.clone(),
11859            )
11860            .await
11861            .expect("typed workflow start");
11862        assert_eq!(
11863            decode_wire_avro_value(
11864                &server.request_body("/api/workflows")["input"],
11865                DEFAULT_CODEC,
11866            )
11867            .expect("typed start input"),
11868            arguments
11869        );
11870
11871        client
11872            .signal_workflow("typed-1", "changed", arguments.clone())
11873            .await
11874            .expect("typed signal");
11875        assert_eq!(
11876            decode_wire_avro_value(
11877                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
11878                DEFAULT_CODEC,
11879            )
11880            .expect("typed signal input"),
11881            arguments
11882        );
11883
11884        assert_eq!(
11885            client
11886                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
11887                .await
11888                .expect("typed query"),
11889            typed_fidelity_probe()
11890        );
11891        assert_eq!(
11892            decode_wire_avro_value(
11893                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
11894                DEFAULT_CODEC,
11895            )
11896            .expect("typed query input"),
11897            arguments
11898        );
11899
11900        assert_eq!(
11901            client
11902                .update_workflow_avro_value(
11903                    "typed-1",
11904                    "replace",
11905                    arguments.clone(),
11906                    Some("typed-request"),
11907                )
11908                .await
11909                .expect("typed update"),
11910            typed_fidelity_probe()
11911        );
11912        let update = server.request_body("/api/workflows/typed-1/update/replace");
11913        assert_eq!(update["request_id"], "typed-request");
11914        assert_eq!(
11915            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
11916            arguments
11917        );
11918
11919        let handle = WorkflowHandle {
11920            client: client.clone(),
11921            workflow_id: "typed-1".to_string(),
11922            run_id: Some("run-typed-1".to_string()),
11923            workflow_type: "typed.echo".to_string(),
11924        };
11925        assert_eq!(
11926            handle
11927                .result_avro_value(WorkflowResultOptions::default())
11928                .await
11929                .expect("typed workflow result"),
11930            typed_fidelity_probe()
11931        );
11932
11933        client
11934            .complete_activity_task(
11935                "activity-typed",
11936                "attempt-typed",
11937                "rust-worker",
11938                typed_fidelity_probe(),
11939                DEFAULT_CODEC,
11940            )
11941            .await
11942            .expect("typed activity completion");
11943        assert_eq!(
11944            decode_wire_avro_value(
11945                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
11946                    ["result"],
11947                DEFAULT_CODEC,
11948            )
11949            .expect("typed activity result"),
11950            typed_fidelity_probe()
11951        );
11952        client
11953            .fail_activity_task(
11954                "activity-typed",
11955                "attempt-typed",
11956                "rust-worker",
11957                "typed failure",
11958                true,
11959            )
11960            .await
11961            .expect("activity failure");
11962    }
11963
11964    #[tokio::test]
11965    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
11966        let server = MockWorkerServer::start();
11967        let client = Client::builder(server.base_url())
11968            .timeout(Duration::from_secs(2))
11969            .build()
11970            .expect("client");
11971
11972        let options = WorkflowCommandOptions::new()
11973            .reason("cleanup requested")
11974            .request_id("cancel-17");
11975        let cancelled = client
11976            .cancel_workflow("wf-lifecycle", options)
11977            .await
11978            .expect("instance cancellation");
11979        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
11980        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
11981        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
11982        assert_eq!(
11983            server.request_body("/api/workflows/wf-lifecycle/cancel"),
11984            json!({"reason":"cleanup requested","request_id":"cancel-17"})
11985        );
11986
11987        let terminated = client
11988            .terminate_workflow(
11989                "wf-lifecycle",
11990                WorkflowCommandOptions::new().reason("forced stop"),
11991            )
11992            .await
11993            .expect("instance termination");
11994        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
11995        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
11996
11997        client
11998            .cancel_workflow_run(
11999                "wf-lifecycle",
12000                "run-current",
12001                WorkflowCommandOptions::default(),
12002            )
12003            .await
12004            .expect("selected run cancellation");
12005        client
12006            .terminate_workflow_run(
12007                "wf-lifecycle",
12008                "run-current",
12009                WorkflowCommandOptions::default(),
12010            )
12011            .await
12012            .expect("selected run termination");
12013
12014        for (command, error) in [
12015            (
12016                WorkflowCommandKind::Cancel,
12017                client
12018                    .cancel_workflow_run(
12019                        "wf-lifecycle",
12020                        "run-stale",
12021                        WorkflowCommandOptions::default(),
12022                    )
12023                    .await
12024                    .expect_err("stale cancellation must be rejected"),
12025            ),
12026            (
12027                WorkflowCommandKind::Terminate,
12028                client
12029                    .terminate_workflow_run(
12030                        "wf-lifecycle",
12031                        "run-stale",
12032                        WorkflowCommandOptions::default(),
12033                    )
12034                    .await
12035                    .expect_err("stale termination must be rejected"),
12036            ),
12037        ] {
12038            let Error::WorkflowCommandRejected(rejection) = error else {
12039                panic!("expected typed command rejection");
12040            };
12041            assert_eq!(rejection.command, command);
12042            assert_eq!(rejection.status, 409);
12043            assert_eq!(rejection.reason, "historical_run_command_rejected");
12044            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
12045            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
12046        }
12047    }
12048
12049    #[tokio::test]
12050    async fn workflow_start_options_send_server_enforced_deadlines() {
12051        let server = MockWorkerServer::start();
12052        let client = Client::builder(server.base_url())
12053            .timeout(Duration::from_secs(2))
12054            .build()
12055            .expect("client");
12056
12057        let handle = client
12058            .start_workflow_with_options(
12059                "rust.timeout",
12060                "rust-timeouts",
12061                "wf-start-options",
12062                WorkflowStartOptions::new()
12063                    .execution_timeout_seconds(30)
12064                    .run_timeout_seconds(1),
12065                json!([]),
12066            )
12067            .await
12068            .expect("workflow start");
12069
12070        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
12071        let body = server.request_body("/api/workflows");
12072        assert_eq!(body["execution_timeout_seconds"], 30);
12073        assert_eq!(body["run_timeout_seconds"], 1);
12074
12075        let invalid = client
12076            .start_workflow_with_options(
12077                "rust.timeout",
12078                "rust-timeouts",
12079                "wf-invalid-options",
12080                WorkflowStartOptions::new()
12081                    .execution_timeout_seconds(1)
12082                    .run_timeout_seconds(2),
12083                json!([]),
12084            )
12085            .await
12086            .expect_err("invalid deadline ordering");
12087        assert!(invalid
12088            .to_string()
12089            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
12090    }
12091
12092    #[tokio::test]
12093    async fn workflow_result_returns_each_typed_terminal_outcome() {
12094        let server = MockWorkerServer::start();
12095        let client = Client::builder(server.base_url())
12096            .timeout(Duration::from_secs(2))
12097            .build()
12098            .expect("client");
12099        let options = WorkflowResultOptions {
12100            poll_interval: Duration::ZERO,
12101            timeout: Duration::from_secs(1),
12102        };
12103
12104        let failed = WorkflowHandle {
12105            client: client.clone(),
12106            workflow_id: "wf-failed".to_string(),
12107            run_id: Some("run-failed".to_string()),
12108            workflow_type: "failure".to_string(),
12109        }
12110        .result(options)
12111        .await
12112        .expect_err("failed outcome");
12113        let Error::WorkflowFailed(failure) = failed else {
12114            panic!("expected WorkflowFailed");
12115        };
12116        assert_eq!(failure.workflow_id, "wf-failed");
12117        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
12118        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
12119        assert_eq!(failure.failure_category.as_deref(), Some("application"));
12120        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
12121        assert_eq!(
12122            failure.exception_class.as_deref(),
12123            Some("billing::PaymentError")
12124        );
12125        assert_eq!(failure.non_retryable, Some(true));
12126
12127        for (workflow_id, expected_kind, expected_reason) in [
12128            (
12129                "wf-cancelled",
12130                WorkflowTerminalKind::Cancelled,
12131                "cleanup requested",
12132            ),
12133            (
12134                "wf-terminated",
12135                WorkflowTerminalKind::Terminated,
12136                "forced stop",
12137            ),
12138            (
12139                "wf-timed-out",
12140                WorkflowTerminalKind::TimedOut,
12141                "run_timeout",
12142            ),
12143        ] {
12144            let error = WorkflowHandle {
12145                client: client.clone(),
12146                workflow_id: workflow_id.to_string(),
12147                run_id: None,
12148                workflow_type: "terminal".to_string(),
12149            }
12150            .result(options)
12151            .await
12152            .expect_err("typed terminal outcome");
12153            let outcome = match error {
12154                Error::WorkflowCancelled(outcome) => outcome,
12155                Error::WorkflowTerminated(outcome) => outcome,
12156                Error::WorkflowTimedOut(outcome) => outcome,
12157                other => panic!("unexpected terminal error: {other}"),
12158            };
12159            assert_eq!(outcome.kind, expected_kind);
12160            assert_eq!(outcome.workflow_id, workflow_id);
12161            assert_eq!(outcome.reason, expected_reason);
12162        }
12163
12164        let wait_timeout = WorkflowHandle {
12165            client,
12166            workflow_id: "wf-waiting".to_string(),
12167            run_id: Some("run-waiting".to_string()),
12168            workflow_type: "waiting".to_string(),
12169        }
12170        .result(WorkflowResultOptions {
12171            poll_interval: Duration::ZERO,
12172            timeout: Duration::ZERO,
12173        })
12174        .await
12175        .expect_err("client wait timeout");
12176        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
12177            panic!("expected typed client timeout");
12178        };
12179        assert_eq!(timeout.reason, "result_wait_timeout");
12180        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
12181        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
12182    }
12183
12184    #[tokio::test]
12185    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
12186        let server = MockWorkerServer::start();
12187        let client = Client::builder(server.base_url())
12188            .timeout(Duration::from_secs(2))
12189            .build()
12190            .expect("client");
12191
12192        let handle = WorkflowHandle {
12193            client,
12194            workflow_id: "wf-selected".to_string(),
12195            run_id: Some("run-selected".to_string()),
12196            workflow_type: "selected".to_string(),
12197        };
12198        let options = WorkflowResultOptions {
12199            poll_interval: Duration::ZERO,
12200            timeout: Duration::from_secs(1),
12201        };
12202
12203        let current = handle
12204            .result(options)
12205            .await
12206            .expect("instance result follows the current run");
12207        assert_eq!(current, json!("current run output"));
12208
12209        let error = handle
12210            .result_selected_run(options)
12211            .await
12212            .expect_err("the selected run is cancelled even though the current run completed");
12213
12214        let Error::WorkflowCancelled(outcome) = error else {
12215            panic!("expected selected run cancellation");
12216        };
12217        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
12218        assert_eq!(outcome.reason, "selected run cancelled");
12219        assert_eq!(
12220            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
12221            1
12222        );
12223        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
12224    }
12225
12226    #[tokio::test]
12227    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
12228        let server = MockWorkerServer::draining_polls();
12229        let client = Client::builder(server.base_url())
12230            .timeout(Duration::from_secs(2))
12231            .build()
12232            .expect("client");
12233
12234        let workflow = client
12235            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
12236            .await
12237            .expect("workflow drain response");
12238        let activity = client
12239            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
12240            .await
12241            .expect("activity drain response");
12242        let query = client
12243            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
12244            .await
12245            .expect("query drain response");
12246
12247        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
12248            assert_eq!(
12249                outcome,
12250                WorkerPollOutcome::Stop {
12251                    poll_status: Some("draining".to_string()),
12252                    reason: Some("worker_draining".to_string()),
12253                }
12254            );
12255        }
12256
12257        assert!(client
12258            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
12259            .await
12260            .expect("compatibility poll")
12261            .is_none());
12262    }
12263
12264    #[tokio::test]
12265    async fn managed_worker_honors_drain_stop_for_every_task_family() {
12266        let server = MockWorkerServer::draining_polls();
12267        let client = Client::builder(server.base_url())
12268            .timeout(Duration::from_secs(2))
12269            .build()
12270            .expect("client");
12271
12272        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
12273            .worker_id("draining-workflow-worker")
12274            .poll_timeout(Duration::ZERO);
12275        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
12276        workflow_worker
12277            .run()
12278            .await
12279            .expect("workflow drain is a clean stop");
12280
12281        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
12282            .worker_id("draining-activity-worker")
12283            .poll_timeout(Duration::ZERO);
12284        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
12285        activity_worker
12286            .run()
12287            .await
12288            .expect("activity drain is a clean stop");
12289
12290        let mut query_worker = Worker::new(client, "rust-workers")
12291            .worker_id("draining-query-worker")
12292            .poll_timeout(Duration::ZERO);
12293        query_worker.register_query("counter", "current", |_ctx, _args| async {
12294            Ok(Value::Null)
12295        });
12296        query_worker
12297            .run()
12298            .await
12299            .expect("query drain is a clean stop");
12300    }
12301
12302    #[tokio::test]
12303    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
12304        let server = MockWorkerServer::start();
12305        let client = Client::builder(server.base_url())
12306            .timeout(Duration::from_secs(2))
12307            .build()
12308            .expect("client");
12309
12310        let heartbeat = client
12311            .heartbeat_activity_task(
12312                "activity-cancel",
12313                "attempt-cancel",
12314                "rust-worker",
12315                typed_fidelity_probe(),
12316            )
12317            .await
12318            .expect("cancellation heartbeat");
12319        assert!(heartbeat.cancel_requested);
12320        assert!(heartbeat.should_stop());
12321        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
12322        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
12323        let heartbeat_body =
12324            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
12325        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
12326        assert_eq!(
12327            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
12328                .expect("typed heartbeat details"),
12329            typed_fidelity_probe()
12330        );
12331
12332        let error = client
12333            .complete_activity_task(
12334                "activity-cancel",
12335                "attempt-cancel",
12336                "rust-worker",
12337                json!({"late":true}),
12338                JSON_CODEC,
12339            )
12340            .await
12341            .expect_err("late completion must be refused");
12342        assert!(activity_task_rejection_is_final(&error));
12343        let Error::ActivityTaskRejected(rejection) = error else {
12344            panic!("expected typed activity rejection");
12345        };
12346        assert_eq!(rejection.status, 409);
12347        assert_eq!(rejection.reason, "run_cancelled");
12348        assert!(rejection.cancel_requested);
12349        assert_eq!(rejection.can_continue, Some(false));
12350    }
12351
12352    #[tokio::test]
12353    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
12354        let server = MockWorkerServer::cancelled_activity();
12355        let client = Client::builder(server.base_url())
12356            .timeout(Duration::from_secs(2))
12357            .build()
12358            .expect("client");
12359        let cancellation_observed = Arc::new(AtomicBool::new(false));
12360        let observed = Arc::clone(&cancellation_observed);
12361        let mut worker = Worker::new(client.clone(), "rust-workers")
12362            .worker_id("rust-cancel-worker")
12363            .poll_timeout(Duration::from_millis(10));
12364        worker.register_activity("cancel-aware", move |ctx, _args| {
12365            let observed = Arc::clone(&observed);
12366            async move {
12367                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
12368                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
12369                Ok(json!({"late":"completion"}))
12370            }
12371        });
12372
12373        assert_eq!(
12374            worker.run_once().await.expect("cancelled attempt handled"),
12375            1
12376        );
12377        assert!(cancellation_observed.load(Ordering::SeqCst));
12378        assert_eq!(
12379            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
12380            1
12381        );
12382
12383        let mut restarted = Worker::new(client, "rust-workers")
12384            .worker_id("rust-cancel-worker-restarted")
12385            .poll_timeout(Duration::from_millis(10));
12386        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
12387        assert_eq!(
12388            restarted
12389                .run_once()
12390                .await
12391                .expect("replacement worker continues polling"),
12392            0
12393        );
12394    }
12395
12396    #[tokio::test]
12397    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
12398        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
12399        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
12400        let client = Client::builder(server.base_url())
12401            .timeout(Duration::from_secs(2))
12402            .build()
12403            .expect("client");
12404
12405        let direct_error = client
12406            .complete_workflow_task(
12407                "workflow-timeout-task",
12408                "timeout-worker",
12409                3,
12410                vec![json!({"type": "complete_workflow", "result": null})],
12411            )
12412            .await
12413            .expect_err("the low-level client preserves the completion rejection");
12414        let Error::Http { status, body } = direct_error else {
12415            panic!("expected the original HTTP completion rejection");
12416        };
12417        assert_eq!(status, reqwest::StatusCode::CONFLICT);
12418        assert_eq!(
12419            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
12420            "run_timed_out"
12421        );
12422
12423        let mut worker = Worker::new(client, "rust-workers")
12424            .worker_id("timeout-worker")
12425            .poll_timeout(Duration::from_millis(10));
12426        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12427            Ok(json!({"late": "result"}))
12428        });
12429
12430        assert_eq!(
12431            worker
12432                .run_once()
12433                .await
12434                .expect("authoritative selected-run timeout settles the tick"),
12435            1
12436        );
12437        assert_eq!(
12438            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
12439            2,
12440            "both the direct client proof and managed worker must see the rejection"
12441        );
12442    }
12443
12444    #[tokio::test]
12445    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
12446        for (name, status, response) in [
12447            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
12448            (
12449                "command was recorded",
12450                "409 Conflict",
12451                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12452            ),
12453            (
12454                "lease conflict",
12455                "409 Conflict",
12456                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
12457            ),
12458            (
12459                "nonterminal run",
12460                "409 Conflict",
12461                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
12462            ),
12463            (
12464                "different selected run",
12465                "409 Conflict",
12466                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
12467            ),
12468            (
12469                "different task attempt",
12470                "409 Conflict",
12471                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12472            ),
12473            (
12474                "authentication failure",
12475                "401 Unauthorized",
12476                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12477            ),
12478            (
12479                "authorization failure",
12480                "403 Forbidden",
12481                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12482            ),
12483            (
12484                "protocol failure",
12485                "400 Bad Request",
12486                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
12487            ),
12488            (
12489                "malformed command",
12490                "422 Unprocessable Entity",
12491                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12492            ),
12493            (
12494                "transient server failure",
12495                "503 Service Unavailable",
12496                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
12497            ),
12498        ] {
12499            let server = MockWorkerServer::workflow_completion(status, response);
12500            let client = Client::builder(server.base_url())
12501                .timeout(Duration::from_secs(2))
12502                .build()
12503                .expect("client");
12504            let mut worker = Worker::new(client, "rust-workers")
12505                .worker_id("timeout-worker")
12506                .poll_timeout(Duration::from_millis(10));
12507            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
12508                Ok(json!({"late": "result"}))
12509            });
12510
12511            let error = worker
12512                .run_once()
12513                .await
12514                .expect_err(&format!("{name} must remain an error"));
12515            assert!(
12516                matches!(error, Error::Http { .. } | Error::Protocol(_)),
12517                "{name} returned an unexpected error variant: {error}"
12518            );
12519        }
12520    }
12521
12522    #[tokio::test]
12523    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
12524        let server = MockWorkerServer::start();
12525        let client = Client::builder(server.base_url())
12526            .worker_token(Some("worker-secret".to_string()))
12527            .namespace("orders")
12528            .timeout(Duration::from_secs(2))
12529            .build()
12530            .expect("client");
12531        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
12532
12533        let result = client
12534            .deregister_worker_registration("worker/α space")
12535            .await
12536            .expect("deregister worker registration");
12537
12538        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
12539        assert_eq!(
12540            server.worker_protocol_for(path).as_deref(),
12541            Some(WORKER_PROTOCOL_VERSION)
12542        );
12543        assert_eq!(server.control_protocol_for(path), None);
12544        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
12545        assert_eq!(
12546            server.authorization_for(path).as_deref(),
12547            Some("Bearer worker-secret")
12548        );
12549        assert_eq!(
12550            result,
12551            WorkerDeregistrationEnvelope {
12552                worker_id: "deregistered-worker".to_string(),
12553                outcome: "deregistered".to_string(),
12554                recovered_workflow_task_count: 2,
12555            }
12556        );
12557    }
12558
12559    #[tokio::test]
12560    async fn low_level_registration_rejects_update_validators_before_transport() {
12561        let server = MockWorkerServer::start();
12562        let client = Client::builder(server.base_url())
12563            .timeout(Duration::from_secs(2))
12564            .build()
12565            .expect("client");
12566
12567        for update_validators in [json!(["approve"]), json!("approve")] {
12568            let error = client
12569                .register_worker_with_command_contracts(
12570                    "validator-claiming-worker",
12571                    "rust-workers",
12572                    vec!["orders".to_string()],
12573                    vec![],
12574                    1,
12575                    1,
12576                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
12577                    json!({
12578                        "orders": {
12579                            "queries": ["current"],
12580                            "updates": ["approve"],
12581                            "update_validators": update_validators,
12582                        },
12583                    }),
12584                )
12585                .await
12586                .expect_err("unsupported validator claims must fail before registration");
12587
12588            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
12589                panic!("expected typed unsupported-validator failure");
12590            };
12591            assert_eq!(workflow_type, "orders");
12592        }
12593        assert_eq!(server.request_count("/api/worker/register"), 0);
12594    }
12595
12596    #[tokio::test]
12597    async fn low_level_registration_preserves_query_and_update_contracts() {
12598        let server = MockWorkerServer::start();
12599        let client = Client::builder(server.base_url())
12600            .timeout(Duration::from_secs(2))
12601            .build()
12602            .expect("client");
12603        let contracts = json!({
12604            "orders": {
12605                "queries": ["current"],
12606                "updates": ["approve"],
12607                "update_validators": [],
12608            },
12609            "payments": {
12610                "queries": ["status"],
12611                "updates": ["capture"],
12612            },
12613        });
12614
12615        client
12616            .register_worker_with_command_contracts(
12617                "command-worker",
12618                "rust-workers",
12619                vec!["orders".to_string(), "payments".to_string()],
12620                vec![],
12621                1,
12622                1,
12623                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
12624                contracts.clone(),
12625            )
12626            .await
12627            .expect("query and update contracts must remain supported");
12628
12629        assert_eq!(
12630            server.request_body("/api/worker/register")["workflow_command_contracts"],
12631            contracts
12632        );
12633    }
12634
12635    #[tokio::test]
12636    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
12637        let server = MockWorkerServer::start();
12638        let control_only = Client::builder(server.base_url())
12639            .control_token(Some("control-secret".to_string()))
12640            .build()
12641            .expect("control client");
12642
12643        let error = control_only
12644            .register_worker("worker", "queue", vec![], vec![], 1, 1)
12645            .await
12646            .expect_err("control token must not authorize a worker request");
12647        assert!(matches!(
12648            error,
12649            Error::MissingRoleCredentials { role: "worker", .. }
12650        ));
12651        assert_eq!(server.request_count("/api/worker/register"), 0);
12652
12653        let worker_only = Client::builder(server.base_url())
12654            .worker_token(Some("worker-secret".to_string()))
12655            .build()
12656            .expect("worker client");
12657        let error = worker_only
12658            .health()
12659            .await
12660            .expect_err("worker token must not authorize a control request");
12661        assert!(matches!(
12662            error,
12663            Error::MissingRoleCredentials {
12664                role: "control",
12665                ..
12666            }
12667        ));
12668        assert_eq!(server.request_count("/api/health"), 0);
12669    }
12670
12671    #[tokio::test]
12672    async fn shared_token_supports_worker_and_control_planes() {
12673        let server = MockWorkerServer::start();
12674        let client = Client::builder(server.base_url())
12675            .token(Some("shared-secret".to_string()))
12676            .build()
12677            .expect("client");
12678
12679        client.health().await.expect("control request");
12680        client
12681            .register_worker("worker", "queue", vec![], vec![], 1, 1)
12682            .await
12683            .expect("worker request");
12684
12685        assert_eq!(
12686            server.authorization_for("/api/health").as_deref(),
12687            Some("Bearer shared-secret")
12688        );
12689        assert_eq!(
12690            server.control_protocol_for("/api/health").as_deref(),
12691            Some(CONTROL_PLANE_VERSION)
12692        );
12693        assert_eq!(
12694            server.authorization_for("/api/worker/register").as_deref(),
12695            Some("Bearer shared-secret")
12696        );
12697        assert_eq!(
12698            server
12699                .worker_protocol_for("/api/worker/register")
12700                .as_deref(),
12701            Some(WORKER_PROTOCOL_VERSION)
12702        );
12703    }
12704
12705    #[tokio::test]
12706    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
12707        let server = MockWorkerServer::start();
12708        let client = Client::builder(server.base_url())
12709            .timeout(Duration::from_secs(2))
12710            .build()
12711            .expect("client");
12712
12713        client
12714            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
12715            .await
12716            .expect("register");
12717        client
12718            .heartbeat_worker("capture-worker", 1, 1)
12719            .await
12720            .expect("heartbeat");
12721        client
12722            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12723            .await
12724            .expect("workflow poll");
12725        client
12726            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12727            .await
12728            .expect("activity poll");
12729
12730        for path in [
12731            "/api/worker/register",
12732            "/api/worker/heartbeat",
12733            "/api/worker/workflow-tasks/poll",
12734            "/api/worker/activity-tasks/poll",
12735        ] {
12736            assert_eq!(
12737                server.worker_protocol_for(path).as_deref(),
12738                Some(WORKER_PROTOCOL_VERSION),
12739                "unexpected protocol for {path}"
12740            );
12741        }
12742
12743        assert_eq!(
12744            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
12745            1
12746        );
12747        assert_eq!(
12748            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
12749            1
12750        );
12751        assert!(
12752            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
12753                .as_str()
12754                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
12755        );
12756        assert!(
12757            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
12758                .as_str()
12759                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
12760        );
12761    }
12762
12763    #[tokio::test]
12764    async fn query_task_endpoints_send_the_query_feature_protocol() {
12765        let server = MockWorkerServer::start();
12766        let client = Client::builder(server.base_url())
12767            .timeout(Duration::from_secs(2))
12768            .build()
12769            .expect("client");
12770
12771        client
12772            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12773            .await
12774            .expect("query poll");
12775        client
12776            .complete_query_task("query-capture", "capture-worker", 1, json!(8), JSON_CODEC)
12777            .await
12778            .expect("query complete");
12779        client
12780            .fail_query_task(
12781                "query-capture",
12782                "capture-worker",
12783                1,
12784                "failed",
12785                "query_rejected",
12786                "QueryFailed",
12787            )
12788            .await
12789            .expect("query fail");
12790
12791        for path in [
12792            "/api/worker/query-tasks/poll",
12793            "/api/worker/query-tasks/query-capture/complete",
12794            "/api/worker/query-tasks/query-capture/fail",
12795        ] {
12796            assert_eq!(
12797                server.worker_protocol_for(path).as_deref(),
12798                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
12799                "unexpected protocol for {path}"
12800            );
12801        }
12802
12803        assert_eq!(
12804            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
12805            1
12806        );
12807        assert!(
12808            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
12809                .as_str()
12810                .is_some_and(|id| id.starts_with("rust-query-poll-"))
12811        );
12812    }
12813
12814    #[tokio::test]
12815    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
12816        let server = MockWorkerServer::transient_worker_failures();
12817        let client = Client::builder(server.base_url())
12818            .timeout(Duration::from_secs(2))
12819            .build()
12820            .expect("client");
12821
12822        client
12823            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
12824            .await
12825            .expect("workflow poll retry");
12826        client
12827            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
12828            .await
12829            .expect("activity poll retry");
12830        client
12831            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12832            .await
12833            .expect("query poll retry");
12834
12835        for path in [
12836            "/api/worker/workflow-tasks/poll",
12837            "/api/worker/activity-tasks/poll",
12838            "/api/worker/query-tasks/poll",
12839        ] {
12840            let bodies = server.request_bodies(path);
12841            assert_eq!(bodies.len(), 2, "{path} must be retried once");
12842            assert_eq!(
12843                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
12844                "{path} must preserve the request binding across retry"
12845            );
12846        }
12847    }
12848
12849    #[tokio::test]
12850    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
12851        let server = MockWorkerServer::consecutive_poll_failures(2);
12852        let client = Client::builder(server.base_url())
12853            .timeout(Duration::from_secs(2))
12854            .build()
12855            .expect("client");
12856        let mut worker = Worker::new(client, "capture")
12857            .worker_id("capture-worker")
12858            .poll_timeout(Duration::from_millis(10))
12859            .retry_policy(WorkerRetryPolicy {
12860                max_retries: 2,
12861                initial_backoff: Duration::from_millis(1),
12862                max_backoff: Duration::from_millis(1),
12863            });
12864        worker.register_workflow(
12865            "capture.workflow",
12866            |_ctx, _input| async move { Ok(Value::Null) },
12867        );
12868        worker.register_activity(
12869            "capture.activity",
12870            |_ctx, _input| async move { Ok(Value::Null) },
12871        );
12872        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
12873            Ok(Value::Null)
12874        });
12875
12876        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
12877
12878        for path in [
12879            "/api/worker/workflow-tasks/poll",
12880            "/api/worker/activity-tasks/poll",
12881            "/api/worker/query-tasks/poll",
12882        ] {
12883            let bodies = server.request_bodies(path);
12884            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
12885            assert!(
12886                bodies
12887                    .iter()
12888                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
12889                "{path} must preserve one request binding across every retry"
12890            );
12891        }
12892    }
12893
12894    #[tokio::test]
12895    async fn query_protocol_rejection_from_older_server_is_typed() {
12896        let server = MockWorkerServer::reject_query_protocol();
12897        let client = Client::builder(server.base_url())
12898            .timeout(Duration::from_secs(2))
12899            .build()
12900            .expect("client");
12901
12902        let error = client
12903            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
12904            .await
12905            .expect_err("server below query protocol floor must reject");
12906        let Error::Protocol(failure) = error else {
12907            panic!("expected typed protocol failure");
12908        };
12909
12910        assert_eq!(failure.status, 400);
12911        assert_eq!(failure.reason, "unsupported_protocol_version");
12912        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
12913        assert_eq!(
12914            failure.requested_version.as_deref(),
12915            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12916        );
12917        assert_eq!(
12918            server
12919                .worker_protocol_for("/api/worker/query-tasks/poll")
12920                .as_deref(),
12921            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
12922        );
12923    }
12924
12925    #[tokio::test]
12926    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
12927        let server = MockWorkerServer::reject_query_protocol();
12928        let client = Client::builder(server.base_url())
12929            .timeout(Duration::from_secs(2))
12930            .build()
12931            .expect("client");
12932        let mut worker = Worker::new(client, "rust-workers")
12933            .worker_id("baseline-worker")
12934            .poll_timeout(Duration::from_millis(10));
12935
12936        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
12937            Ok(Value::Null)
12938        });
12939
12940        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
12941        assert_eq!(
12942            server
12943                .worker_protocol_for("/api/worker/workflow-tasks/poll")
12944                .as_deref(),
12945            Some(WORKER_PROTOCOL_VERSION)
12946        );
12947        assert_eq!(
12948            server.worker_protocol_for("/api/worker/query-tasks/poll"),
12949            None,
12950            "a worker without query handlers must not use the query-task endpoint"
12951        );
12952    }
12953
12954    #[tokio::test]
12955    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
12956        let server = MockWorkerServer::reject_query_completion();
12957        let client = Client::builder(server.base_url())
12958            .timeout(Duration::from_secs(2))
12959            .build()
12960            .expect("client");
12961
12962        let error = client
12963            .complete_query_task("query-late", "late-worker", 1, json!(8), JSON_CODEC)
12964            .await
12965            .expect_err("expired completion must be rejected");
12966        let Error::QueryFailed(failure) = error else {
12967            panic!("expected typed query failure");
12968        };
12969        assert_eq!(failure.status, 409);
12970        assert_eq!(failure.reason, "query_task_timed_out");
12971
12972        let mut worker = Worker::new(client, "rust-workers")
12973            .worker_id("late-worker")
12974            .poll_timeout(Duration::from_millis(10));
12975        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
12976        worker.register_query(
12977            "counter",
12978            "current",
12979            |_ctx, _args| async move { Ok(json!(8)) },
12980        );
12981
12982        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
12983        assert_eq!(
12984            worker
12985                .run_once()
12986                .await
12987                .expect("worker continues after late completion"),
12988            0
12989        );
12990        assert_eq!(
12991            server.request_count("/api/worker/query-tasks/query-late/complete"),
12992            2
12993        );
12994        assert_eq!(
12995            server.request_count("/api/worker/query-tasks/query-late/fail"),
12996            0,
12997            "a server completion rejection must not be reported as an encoding failure"
12998        );
12999    }
13000
13001    #[tokio::test]
13002    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
13003        let server = MockWorkerServer::start();
13004        let client = Client::builder(server.base_url())
13005            .timeout(Duration::from_secs(2))
13006            .build()
13007            .expect("client");
13008        let mut worker = Worker::new(client, "rust-workers")
13009            .worker_id("joined-worker")
13010            .poll_timeout(Duration::from_millis(10));
13011        worker.register_workflow(
13012            "joined.workflow",
13013            |_ctx, _input| async move { Ok(Value::Null) },
13014        );
13015        worker.register_activity(
13016            "joined.activity",
13017            |_ctx, _input| async move { Ok(Value::Null) },
13018        );
13019        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
13020            Ok(Value::Null)
13021        });
13022
13023        worker
13024            .run_until(tokio::time::sleep(Duration::from_millis(20)))
13025            .await
13026            .expect("normal shutdown");
13027
13028        let deregistration_path = "/api/worker/registrations/mock-worker";
13029        assert_eq!(server.request_count(deregistration_path), 1);
13030        for poll_path in [
13031            "/api/worker/workflow-tasks/poll",
13032            "/api/worker/activity-tasks/poll",
13033            "/api/worker/query-tasks/poll",
13034        ] {
13035            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
13036        }
13037        assert_eq!(
13038            server.captured_paths().last().map(String::as_str),
13039            Some(deregistration_path),
13040            "deregistration must start only after every poller has joined"
13041        );
13042    }
13043
13044    #[tokio::test]
13045    async fn registration_failure_does_not_deregister() {
13046        let server = MockWorkerServer::rejected_registration();
13047        let client = Client::builder(server.base_url())
13048            .timeout(Duration::from_secs(2))
13049            .build()
13050            .expect("client");
13051        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
13052
13053        let error = worker
13054            .run_until(async {})
13055            .await
13056            .expect_err("registration must fail");
13057        assert!(matches!(
13058            error,
13059            Error::Http {
13060                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
13061                ..
13062            }
13063        ));
13064        assert!(server
13065            .captured_paths()
13066            .iter()
13067            .all(|path| !path.starts_with("/api/worker/registrations/")));
13068    }
13069
13070    #[tokio::test]
13071    async fn declined_registration_does_not_deregister() {
13072        let server = MockWorkerServer::declined_registration();
13073        let client = Client::builder(server.base_url())
13074            .timeout(Duration::from_secs(2))
13075            .build()
13076            .expect("client");
13077        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
13078
13079        let error = worker
13080            .run_until(async {})
13081            .await
13082            .expect_err("declined registration must fail");
13083        assert!(matches!(error, Error::WorkerLoop(_)));
13084        assert!(error.to_string().contains("was not accepted"));
13085        assert!(server
13086            .captured_paths()
13087            .iter()
13088            .all(|path| !path.starts_with("/api/worker/registrations/")));
13089    }
13090
13091    #[tokio::test]
13092    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
13093        let server = MockWorkerServer::rejected_deregistration();
13094        let client = Client::builder(server.base_url())
13095            .timeout(Duration::from_secs(2))
13096            .build()
13097            .expect("client");
13098        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
13099
13100        let error = worker
13101            .run_until(async {})
13102            .await
13103            .expect_err("deregistration must fail");
13104        assert!(matches!(
13105            error,
13106            Error::Http {
13107                status: reqwest::StatusCode::FORBIDDEN,
13108                ..
13109            }
13110        ));
13111        assert_eq!(
13112            server.request_count("/api/worker/registrations/mock-worker"),
13113            1
13114        );
13115    }
13116
13117    #[tokio::test]
13118    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
13119        let server = MockWorkerServer::rejected_deregistration_protocol();
13120        let client = Client::builder(server.base_url())
13121            .timeout(Duration::from_secs(2))
13122            .build()
13123            .expect("client");
13124        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
13125
13126        let error = worker
13127            .run_until(async {})
13128            .await
13129            .expect_err("protocol rejection must fail shutdown");
13130        let Error::Protocol(failure) = error else {
13131            panic!("expected typed protocol failure");
13132        };
13133        assert_eq!(failure.reason, "unsupported_protocol_version");
13134        assert_eq!(failure.requested_version.as_deref(), Some("1.2"));
13135        assert_eq!(
13136            server.request_count("/api/worker/registrations/mock-worker"),
13137            1
13138        );
13139    }
13140
13141    #[tokio::test]
13142    async fn primary_poller_error_retains_deregistration_failure_context() {
13143        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
13144        let client = Client::builder(server.base_url())
13145            .timeout(Duration::from_secs(2))
13146            .build()
13147            .expect("client");
13148        let mut worker = Worker::new(client, "rust-workers")
13149            .worker_id("combined-failure")
13150            .poll_timeout(Duration::from_millis(10));
13151        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
13152            Ok(Value::Null)
13153        });
13154
13155        let error = worker
13156            .run()
13157            .await
13158            .expect_err("worker and cleanup must fail");
13159        let summary = error.to_string();
13160        assert!(summary.contains("authentication_failed"));
13161        assert!(summary.contains("worker cannot deregister"));
13162        let Error::WorkerShutdown {
13163            primary,
13164            deregistration,
13165        } = error
13166        else {
13167            panic!("expected combined worker shutdown error");
13168        };
13169        assert!(matches!(
13170            *primary,
13171            Error::Http {
13172                status: reqwest::StatusCode::UNAUTHORIZED,
13173                ..
13174            }
13175        ));
13176        assert!(matches!(
13177            *deregistration,
13178            Error::Http {
13179                status: reqwest::StatusCode::FORBIDDEN,
13180                ..
13181            }
13182        ));
13183        assert_eq!(
13184            server.request_count("/api/worker/registrations/mock-worker"),
13185            1
13186        );
13187    }
13188
13189    #[tokio::test]
13190    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
13191        let server = MockWorkerServer::start();
13192        let client = Client::builder(server.base_url())
13193            .timeout(Duration::from_secs(2))
13194            .build()
13195            .expect("client");
13196        let mut worker = Worker::new(client, "rust-workers")
13197            .worker_id("activity-only-worker")
13198            .poll_timeout(Duration::from_millis(10));
13199
13200        worker.register_activity(
13201            "activity.only",
13202            |_ctx, _args| async move { Ok(Value::Null) },
13203        );
13204
13205        worker.run_until(async {}).await.expect("run worker");
13206    }
13207
13208    #[tokio::test]
13209    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
13210        let server = MockWorkerServer::start();
13211        let client = Client::builder(server.base_url())
13212            .timeout(Duration::from_secs(2))
13213            .build()
13214            .expect("client");
13215        let mut worker = Worker::new(client, "rust-workers")
13216            .worker_id("workflow-only-worker")
13217            .poll_timeout(Duration::from_millis(10));
13218
13219        worker.register_workflow(
13220            "workflow.only",
13221            |_ctx, _input| async move { Ok(Value::Null) },
13222        );
13223
13224        worker.run_until(async {}).await.expect("run worker");
13225    }
13226
13227    #[tokio::test]
13228    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
13229        let server = MockWorkerServer::start();
13230        let client = Client::builder(server.base_url())
13231            .timeout(Duration::from_secs(2))
13232            .build()
13233            .expect("client");
13234        let observations = Arc::new(Mutex::new(Vec::new()));
13235        let observed = Arc::clone(&observations);
13236        let mut worker = Worker::new(client, "rust-workers")
13237            .worker_id("observed-heartbeat-worker")
13238            .poll_timeout(Duration::from_millis(10))
13239            .on_worker_heartbeat(move |observation| {
13240                observed
13241                    .lock()
13242                    .expect("heartbeat observations")
13243                    .push(observation.clone());
13244            });
13245
13246        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
13247            Ok(Value::Null)
13248        });
13249        let acknowledged = Arc::clone(&observations);
13250        worker
13251            .run_until(async move {
13252                tokio::time::timeout(Duration::from_secs(2), async move {
13253                    loop {
13254                        if !acknowledged
13255                            .lock()
13256                            .expect("heartbeat observations")
13257                            .is_empty()
13258                        {
13259                            break;
13260                        }
13261                        tokio::time::sleep(Duration::from_millis(1)).await;
13262                    }
13263                })
13264                .await
13265                .expect("heartbeat acknowledgement within timeout");
13266            })
13267            .await
13268            .expect("run worker");
13269
13270        let observations = observations.lock().expect("heartbeat observations");
13271        let first = observations.first().expect("heartbeat acknowledgement");
13272        assert_eq!(first.worker_id, "observed-heartbeat-worker");
13273        assert_eq!(first.task_queue, "rust-workers");
13274        assert!(first.acknowledged_at_unix_millis > 0);
13275        assert_eq!(first.acknowledgement, json!({}));
13276    }
13277
13278    #[tokio::test]
13279    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
13280        let server = MockWorkerServer::delayed_heartbeat_worker();
13281        let client = Client::builder(server.base_url())
13282            .timeout(Duration::from_secs(3))
13283            .build()
13284            .expect("client");
13285        let observations = Arc::new(Mutex::new(Vec::new()));
13286        let observed = Arc::clone(&observations);
13287        let mut worker = Worker::new(client, "rust-snapshot-workers")
13288            .worker_id("rust-snapshot-worker")
13289            .poll_timeout(Duration::from_millis(10))
13290            .on_worker_heartbeat(move |observation| {
13291                observed
13292                    .lock()
13293                    .expect("heartbeat observations")
13294                    .push(observation.clone());
13295            });
13296
13297        worker.register_workflow("snapshot", |ctx, _input| async move {
13298            ctx.wait_signal("finish").await?;
13299            Ok(json!({"status": "finished"}))
13300        });
13301        worker.register_query("snapshot", "current", |ctx, _args| async move {
13302            Ok(json!(ctx
13303                .signals("increment")
13304                .iter()
13305                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13306                .sum::<i64>()))
13307        });
13308        worker.register_activity("cancel-aware", |_ctx, _args| async move {
13309            Ok(json!({"late": "completion"}))
13310        });
13311
13312        worker
13313            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
13314            .await
13315            .expect("delayed heartbeat must allow a clean worker shutdown");
13316
13317        let observations = observations.lock().expect("heartbeat observations");
13318        assert!(
13319            observations.len() >= 3,
13320            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
13321        );
13322        assert!(
13323            observations.windows(2).all(|pair| {
13324                pair[1].acknowledged_at_unix_millis
13325                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
13326                    >= 850
13327            }),
13328            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
13329        );
13330        drop(observations);
13331
13332        let heartbeat_times = server.request_times("/api/worker/heartbeat");
13333        let delayed_request_at = *heartbeat_times
13334            .get(1)
13335            .expect("intentionally delayed heartbeat request");
13336        let delay_window_start = delayed_request_at + Duration::from_millis(100);
13337        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
13338        for path in [
13339            "/api/worker/workflow-tasks/poll",
13340            "/api/worker/activity-tasks/poll",
13341            "/api/worker/query-tasks/poll",
13342        ] {
13343            assert!(
13344                server
13345                    .request_times(path)
13346                    .iter()
13347                    .any(|received_at| *received_at >= delay_window_start
13348                        && *received_at <= delay_window_end),
13349                "{path} must keep polling while a heartbeat acknowledgement is delayed"
13350            );
13351        }
13352        assert!(
13353            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
13354            "workflow work must be settled"
13355        );
13356        assert!(
13357            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
13358            "activity work must be settled"
13359        );
13360        assert!(
13361            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
13362            "query work must be settled"
13363        );
13364    }
13365
13366    #[tokio::test]
13367    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
13368        let server = MockWorkerServer::heartbeat_retry_worker();
13369        let client = Client::builder(server.base_url())
13370            .timeout(Duration::from_secs(2))
13371            .build()
13372            .expect("client");
13373        let observations = Arc::new(Mutex::new(Vec::new()));
13374        let observed = Arc::clone(&observations);
13375        let worker = Worker::new(client, "rust-workers")
13376            .worker_id("heartbeat-retry-worker")
13377            .retry_policy(WorkerRetryPolicy {
13378                max_retries: 1,
13379                initial_backoff: Duration::from_millis(300),
13380                max_backoff: Duration::from_millis(300),
13381            })
13382            .on_worker_heartbeat(move |observation| {
13383                observed
13384                    .lock()
13385                    .expect("heartbeat observations")
13386                    .push(observation.clone());
13387            });
13388
13389        worker
13390            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
13391            .await
13392            .expect("retryable heartbeat failure must remain bounded and recover");
13393
13394        let observations = observations.lock().expect("heartbeat observations");
13395        assert!(observations.len() >= 3, "heartbeat retry must recover");
13396        assert!(
13397            observations.windows(2).all(|pair| {
13398                pair[1]
13399                    .acknowledged_at_unix_millis
13400                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
13401                    >= 850
13402            }),
13403            "a successful retry must start a fresh advertised cadence: {observations:?}"
13404        );
13405        assert_eq!(
13406            server.request_count("/api/worker/heartbeat"),
13407            observations.len() + 1,
13408            "one retryable failure must add exactly one bounded request"
13409        );
13410    }
13411
13412    #[tokio::test]
13413    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
13414        let server = MockWorkerServer::waiting_query_worker();
13415        let client = Client::builder(server.base_url())
13416            .timeout(Duration::from_secs(2))
13417            .build()
13418            .expect("client");
13419        let observations = Arc::new(Mutex::new(Vec::new()));
13420        let observed = Arc::clone(&observations);
13421        let mut worker = Worker::new(client, "rust-snapshot-workers")
13422            .worker_id("rust-snapshot-worker")
13423            .poll_timeout(Duration::from_millis(10))
13424            .on_worker_heartbeat(move |observation| {
13425                observed
13426                    .lock()
13427                    .expect("heartbeat observations")
13428                    .push(observation.clone());
13429            });
13430
13431        worker.register_workflow("snapshot", |ctx, _input| async move {
13432            ctx.wait_signal("finish").await?;
13433            Ok(json!({"status": "finished"}))
13434        });
13435        worker.register_query("snapshot", "current", |ctx, _args| async move {
13436            let current = ctx
13437                .signals("increment")
13438                .iter()
13439                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
13440                .sum::<i64>();
13441            Ok(json!(current))
13442        });
13443        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
13444
13445        worker
13446            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
13447            .await
13448            .expect("pending workflow and query poller must remain live until shutdown");
13449
13450        assert!(
13451            observations.lock().expect("heartbeat observations").len() >= 4,
13452            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
13453        );
13454        assert!(
13455            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
13456            "workflow polling must continue after empty replay acknowledgements"
13457        );
13458        assert!(
13459            server.request_count("/api/worker/query-tasks/poll") >= 2,
13460            "query polling must continue after serving the current query"
13461        );
13462        assert_eq!(
13463            server.request_body("/api/worker/register")["capabilities"],
13464            json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
13465        );
13466        assert_eq!(
13467            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
13468            json!({
13469                "queries": ["current"],
13470                "updates": ["replace"],
13471                "update_validators": [],
13472            })
13473        );
13474
13475        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
13476        assert_eq!(
13477            opened["commands"],
13478            json!([{
13479                "type": "open_signal_wait",
13480                "signal_name": "finish",
13481            }])
13482        );
13483
13484        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
13485            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
13486            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
13487            let failure = server.request_body(&fail_path);
13488            assert_eq!(
13489                failure["failure"]["type"],
13490                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
13491            );
13492            assert_eq!(server.request_count(&completion_path), 0);
13493        }
13494
13495        let query_completion =
13496            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
13497        assert_eq!(query_completion["result"], json!(8));
13498
13499        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
13500        assert_eq!(
13501            server.request_count(terminal_path),
13502            1,
13503            "the matching signal must settle the workflow exactly once"
13504        );
13505        let terminal = server.request_body(terminal_path);
13506        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
13507        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
13508        assert_eq!(
13509            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
13510                .expect("terminal workflow result"),
13511            json!({"status": "finished"})
13512        );
13513    }
13514
13515    #[tokio::test]
13516    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
13517        let server = MockWorkerServer::transient_worker_failures();
13518        let client = Client::builder(server.base_url())
13519            .timeout(Duration::from_secs(2))
13520            .build()
13521            .expect("client");
13522        let mut worker = Worker::new(client, "rust-workers")
13523            .worker_id("retry-worker")
13524            .poll_timeout(Duration::from_millis(10))
13525            .retry_policy(WorkerRetryPolicy {
13526                max_retries: 2,
13527                initial_backoff: Duration::from_millis(1),
13528                max_backoff: Duration::from_millis(1),
13529            });
13530        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13531        worker.register_activity(
13532            "counter.activity",
13533            |_ctx, _input| async move { Ok(Value::Null) },
13534        );
13535        worker.register_query(
13536            "counter",
13537            "current",
13538            |_ctx, _args| async move { Ok(json!(8)) },
13539        );
13540
13541        worker
13542            .run_until(tokio::time::sleep(Duration::from_millis(75)))
13543            .await
13544            .expect("transient failures must not stop the worker");
13545
13546        for path in [
13547            "/api/worker/heartbeat",
13548            "/api/worker/workflow-tasks/poll",
13549            "/api/worker/activity-tasks/poll",
13550            "/api/worker/query-tasks/poll",
13551        ] {
13552            assert!(
13553                server.request_count(path) >= 2,
13554                "{path} must continue after its transient failure"
13555            );
13556        }
13557    }
13558
13559    #[tokio::test]
13560    async fn worker_bounds_transport_retries() {
13561        let server = MockWorkerServer::unavailable_polls();
13562        let client = Client::builder(server.base_url())
13563            .timeout(Duration::from_secs(2))
13564            .build()
13565            .expect("client");
13566        let mut worker = Worker::new(client, "rust-workers")
13567            .worker_id("bounded-retry-worker")
13568            .poll_timeout(Duration::from_millis(10))
13569            .retry_policy(WorkerRetryPolicy {
13570                max_retries: 2,
13571                initial_backoff: Duration::from_millis(1),
13572                max_backoff: Duration::from_millis(1),
13573            });
13574        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13575
13576        let error = worker.run().await.expect_err("retry bound must terminate");
13577        assert!(matches!(error, Error::Transport(_)));
13578        assert_eq!(
13579            server.request_count("/api/worker/workflow-tasks/poll"),
13580            3,
13581            "one initial request plus exactly two retries"
13582        );
13583    }
13584
13585    #[tokio::test]
13586    async fn worker_retry_policy_can_disable_poll_retries() {
13587        let server = MockWorkerServer::unavailable_polls();
13588        let client = Client::builder(server.base_url())
13589            .timeout(Duration::from_secs(2))
13590            .build()
13591            .expect("client");
13592        let mut worker = Worker::new(client, "rust-workers")
13593            .worker_id("no-retry-worker")
13594            .poll_timeout(Duration::from_millis(10))
13595            .retry_policy(WorkerRetryPolicy {
13596                max_retries: 0,
13597                initial_backoff: Duration::from_millis(1),
13598                max_backoff: Duration::from_millis(1),
13599            });
13600        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13601
13602        let error = worker
13603            .run_once()
13604            .await
13605            .expect_err("disabled retries must return the first transport failure");
13606        assert!(matches!(error, Error::Transport(_)));
13607        assert_eq!(
13608            server.request_count("/api/worker/workflow-tasks/poll"),
13609            1,
13610            "max_retries=0 must send only the initial request"
13611        );
13612    }
13613
13614    #[tokio::test]
13615    async fn worker_does_not_retry_authentication_failures() {
13616        let server = MockWorkerServer::unauthorized_polls();
13617        let client = Client::builder(server.base_url())
13618            .timeout(Duration::from_secs(2))
13619            .build()
13620            .expect("client");
13621        let mut worker = Worker::new(client, "rust-workers")
13622            .worker_id("unauthorized-worker")
13623            .poll_timeout(Duration::from_millis(10));
13624        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
13625
13626        let error = worker
13627            .run()
13628            .await
13629            .expect_err("authentication must terminate");
13630        let Error::Http { status, body } = error else {
13631            panic!("expected stable HTTP authentication error");
13632        };
13633        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
13634        assert!(body.contains("authentication_failed"));
13635        assert_eq!(
13636            server.request_count("/api/worker/workflow-tasks/poll"),
13637            1,
13638            "authentication failures must not be retried"
13639        );
13640    }
13641
13642    #[derive(Clone, Debug)]
13643    struct CapturedRequest {
13644        method: String,
13645        path: String,
13646        authorization: Option<String>,
13647        namespace: Option<String>,
13648        worker_protocol: Option<String>,
13649        control_protocol: Option<String>,
13650        body: String,
13651        received_at: Instant,
13652    }
13653
13654    struct MockWorkerServer {
13655        addr: SocketAddr,
13656        stop: Arc<AtomicBool>,
13657        requests: Arc<Mutex<Vec<CapturedRequest>>>,
13658        thread: Option<thread::JoinHandle<()>>,
13659    }
13660
13661    #[derive(Clone, Copy, Default)]
13662    struct MockWorkerBehavior {
13663        reject_query_protocol: bool,
13664        reject_query_completion: bool,
13665        waiting_query_worker: bool,
13666        decline_registration: bool,
13667        complete_named_signal: bool,
13668        poll_failures_per_path: usize,
13669        heartbeat_failures: usize,
13670        heartbeat_failure_request: Option<usize>,
13671        delayed_heartbeat_request: Option<usize>,
13672        heartbeat_response_delay: Duration,
13673        concurrent_requests: bool,
13674        unauthorized_polls: bool,
13675        reject_registration: bool,
13676        reject_deregistration: bool,
13677        reject_deregistration_protocol: bool,
13678        cancelled_activity: bool,
13679        draining_polls: bool,
13680        workflow_completion_status: Option<&'static str>,
13681        workflow_completion_body: Option<&'static str>,
13682    }
13683
13684    impl MockWorkerServer {
13685        fn start() -> Self {
13686            Self::start_with_behavior(MockWorkerBehavior::default())
13687        }
13688
13689        fn reject_query_protocol() -> Self {
13690            Self::start_with_behavior(MockWorkerBehavior {
13691                reject_query_protocol: true,
13692                ..MockWorkerBehavior::default()
13693            })
13694        }
13695
13696        fn reject_query_completion() -> Self {
13697            Self::start_with_behavior(MockWorkerBehavior {
13698                reject_query_completion: true,
13699                ..MockWorkerBehavior::default()
13700            })
13701        }
13702
13703        fn waiting_query_worker() -> Self {
13704            Self::start_with_behavior(MockWorkerBehavior {
13705                waiting_query_worker: true,
13706                complete_named_signal: true,
13707                ..MockWorkerBehavior::default()
13708            })
13709        }
13710
13711        fn transient_worker_failures() -> Self {
13712            Self::start_with_behavior(MockWorkerBehavior {
13713                poll_failures_per_path: 1,
13714                heartbeat_failures: 1,
13715                ..MockWorkerBehavior::default()
13716            })
13717        }
13718
13719        fn consecutive_poll_failures(count: usize) -> Self {
13720            Self::start_with_behavior(MockWorkerBehavior {
13721                poll_failures_per_path: count,
13722                ..MockWorkerBehavior::default()
13723            })
13724        }
13725
13726        fn delayed_heartbeat_worker() -> Self {
13727            Self::start_with_behavior(MockWorkerBehavior {
13728                waiting_query_worker: true,
13729                delayed_heartbeat_request: Some(2),
13730                heartbeat_response_delay: Duration::from_millis(1_500),
13731                concurrent_requests: true,
13732                cancelled_activity: true,
13733                ..MockWorkerBehavior::default()
13734            })
13735        }
13736
13737        fn heartbeat_retry_worker() -> Self {
13738            Self::start_with_behavior(MockWorkerBehavior {
13739                waiting_query_worker: true,
13740                heartbeat_failure_request: Some(2),
13741                concurrent_requests: true,
13742                ..MockWorkerBehavior::default()
13743            })
13744        }
13745
13746        fn unavailable_polls() -> Self {
13747            Self::start_with_behavior(MockWorkerBehavior {
13748                poll_failures_per_path: usize::MAX,
13749                ..MockWorkerBehavior::default()
13750            })
13751        }
13752
13753        fn unauthorized_polls() -> Self {
13754            Self::start_with_behavior(MockWorkerBehavior {
13755                unauthorized_polls: true,
13756                ..MockWorkerBehavior::default()
13757            })
13758        }
13759
13760        fn rejected_registration() -> Self {
13761            Self::start_with_behavior(MockWorkerBehavior {
13762                reject_registration: true,
13763                ..MockWorkerBehavior::default()
13764            })
13765        }
13766
13767        fn declined_registration() -> Self {
13768            Self::start_with_behavior(MockWorkerBehavior {
13769                decline_registration: true,
13770                ..MockWorkerBehavior::default()
13771            })
13772        }
13773
13774        fn rejected_deregistration() -> Self {
13775            Self::start_with_behavior(MockWorkerBehavior {
13776                reject_deregistration: true,
13777                ..MockWorkerBehavior::default()
13778            })
13779        }
13780
13781        fn rejected_deregistration_protocol() -> Self {
13782            Self::start_with_behavior(MockWorkerBehavior {
13783                reject_deregistration_protocol: true,
13784                ..MockWorkerBehavior::default()
13785            })
13786        }
13787
13788        fn unauthorized_polls_and_rejected_deregistration() -> Self {
13789            Self::start_with_behavior(MockWorkerBehavior {
13790                unauthorized_polls: true,
13791                reject_deregistration: true,
13792                ..MockWorkerBehavior::default()
13793            })
13794        }
13795
13796        fn cancelled_activity() -> Self {
13797            Self::start_with_behavior(MockWorkerBehavior {
13798                cancelled_activity: true,
13799                ..MockWorkerBehavior::default()
13800            })
13801        }
13802
13803        fn draining_polls() -> Self {
13804            Self::start_with_behavior(MockWorkerBehavior {
13805                draining_polls: true,
13806                ..MockWorkerBehavior::default()
13807            })
13808        }
13809
13810        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
13811            Self::start_with_behavior(MockWorkerBehavior {
13812                workflow_completion_status: Some(status),
13813                workflow_completion_body: Some(body),
13814                ..MockWorkerBehavior::default()
13815            })
13816        }
13817
13818        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
13819            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
13820            listener
13821                .set_nonblocking(true)
13822                .expect("configure mock listener");
13823            let addr = listener.local_addr().expect("mock server address");
13824            let stop = Arc::new(AtomicBool::new(false));
13825            let server_stop = Arc::clone(&stop);
13826            let requests = Arc::new(Mutex::new(Vec::new()));
13827            let server_requests = Arc::clone(&requests);
13828            let thread = thread::spawn(move || {
13829                let mut request_threads = Vec::new();
13830                while !server_stop.load(Ordering::SeqCst) {
13831                    match listener.accept() {
13832                        Ok((mut stream, _)) => {
13833                            if behavior.concurrent_requests {
13834                                let requests = Arc::clone(&server_requests);
13835                                request_threads.push(thread::spawn(move || {
13836                                    handle_mock_worker_request(&mut stream, &requests, behavior)
13837                                }));
13838                            } else {
13839                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
13840                            }
13841                        }
13842                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
13843                            let mut index = 0;
13844                            while index < request_threads.len() {
13845                                if request_threads[index].is_finished() {
13846                                    request_threads
13847                                        .swap_remove(index)
13848                                        .join()
13849                                        .expect("join mock request");
13850                                } else {
13851                                    index += 1;
13852                                }
13853                            }
13854                            thread::sleep(Duration::from_millis(5));
13855                        }
13856                        Err(_) => break,
13857                    }
13858                }
13859                for request_thread in request_threads {
13860                    request_thread.join().expect("join mock request");
13861                }
13862            });
13863
13864            Self {
13865                addr,
13866                stop,
13867                requests,
13868                thread: Some(thread),
13869            }
13870        }
13871
13872        fn base_url(&self) -> String {
13873            format!("http://{}", self.addr)
13874        }
13875
13876        fn worker_protocol_for(&self, path: &str) -> Option<String> {
13877            self.requests
13878                .lock()
13879                .expect("captured requests")
13880                .iter()
13881                .find(|request| request.path == path)
13882                .and_then(|request| request.worker_protocol.clone())
13883        }
13884
13885        fn control_protocol_for(&self, path: &str) -> Option<String> {
13886            self.requests
13887                .lock()
13888                .expect("captured requests")
13889                .iter()
13890                .find(|request| request.path == path)
13891                .and_then(|request| request.control_protocol.clone())
13892        }
13893
13894        fn method_for(&self, path: &str) -> Option<String> {
13895            self.requests
13896                .lock()
13897                .expect("captured requests")
13898                .iter()
13899                .find(|request| request.path == path)
13900                .map(|request| request.method.clone())
13901        }
13902
13903        fn authorization_for(&self, path: &str) -> Option<String> {
13904            self.requests
13905                .lock()
13906                .expect("captured requests")
13907                .iter()
13908                .find(|request| request.path == path)
13909                .and_then(|request| request.authorization.clone())
13910        }
13911
13912        fn namespace_for(&self, path: &str) -> Option<String> {
13913            self.requests
13914                .lock()
13915                .expect("captured requests")
13916                .iter()
13917                .find(|request| request.path == path)
13918                .and_then(|request| request.namespace.clone())
13919        }
13920
13921        fn request_count(&self, path: &str) -> usize {
13922            self.requests
13923                .lock()
13924                .expect("captured requests")
13925                .iter()
13926                .filter(|request| request.path == path)
13927                .count()
13928        }
13929
13930        fn captured_paths(&self) -> Vec<String> {
13931            self.requests
13932                .lock()
13933                .expect("captured requests")
13934                .iter()
13935                .map(|request| request.path.clone())
13936                .collect()
13937        }
13938
13939        fn request_times(&self, path: &str) -> Vec<Instant> {
13940            self.requests
13941                .lock()
13942                .expect("captured requests")
13943                .iter()
13944                .filter(|request| request.path == path)
13945                .map(|request| request.received_at)
13946                .collect()
13947        }
13948
13949        fn request_body(&self, path: &str) -> Value {
13950            let requests = self.requests.lock().expect("captured requests");
13951            let body = &requests
13952                .iter()
13953                .find(|request| request.path == path)
13954                .unwrap_or_else(|| panic!("missing request for {path}"))
13955                .body;
13956            serde_json::from_str(body).unwrap_or_else(|error| {
13957                panic!("invalid JSON request body for {path}: {error}: {body:?}")
13958            })
13959        }
13960
13961        fn request_bodies(&self, path: &str) -> Vec<Value> {
13962            self.requests
13963                .lock()
13964                .expect("captured requests")
13965                .iter()
13966                .filter(|request| request.path == path)
13967                .map(|request| {
13968                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
13969                        panic!(
13970                            "invalid JSON request body for {path}: {error}: {:?}",
13971                            request.body
13972                        )
13973                    })
13974                })
13975                .collect()
13976        }
13977    }
13978
13979    impl Drop for MockWorkerServer {
13980        fn drop(&mut self) {
13981            self.stop.store(true, Ordering::SeqCst);
13982            let _ = TcpStream::connect(self.addr);
13983
13984            if let Some(thread) = self.thread.take() {
13985                thread.join().expect("join mock server");
13986            }
13987        }
13988    }
13989
13990    fn handle_mock_worker_request(
13991        stream: &mut TcpStream,
13992        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
13993        behavior: MockWorkerBehavior,
13994    ) {
13995        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
13996        let mut buffer = [0_u8; 8192];
13997        let mut request = Vec::new();
13998
13999        loop {
14000            match stream.read(&mut buffer) {
14001                Ok(0) => break,
14002                Ok(read) => {
14003                    request.extend_from_slice(&buffer[..read]);
14004                    if mock_request_is_complete(&request) {
14005                        break;
14006                    }
14007                }
14008                Err(error)
14009                    if matches!(
14010                        error.kind(),
14011                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
14012                    ) =>
14013                {
14014                    break;
14015                }
14016                Err(_) => return,
14017            }
14018        }
14019
14020        let request = String::from_utf8_lossy(&request);
14021        let body = request
14022            .split_once("\r\n\r\n")
14023            .map(|(_, body)| body)
14024            .unwrap_or_default();
14025        let path = request
14026            .lines()
14027            .next()
14028            .and_then(|line| line.split_whitespace().nth(1))
14029            .unwrap_or_default();
14030        let method = request
14031            .lines()
14032            .next()
14033            .and_then(|line| line.split_whitespace().next())
14034            .unwrap_or_default();
14035        let authorization = request.lines().find_map(|line| {
14036            let (name, value) = line.split_once(':')?;
14037            name.eq_ignore_ascii_case("Authorization")
14038                .then(|| value.trim().to_string())
14039        });
14040        let namespace = request.lines().find_map(|line| {
14041            let (name, value) = line.split_once(':')?;
14042            name.eq_ignore_ascii_case("X-Namespace")
14043                .then(|| value.trim().to_string())
14044        });
14045        let worker_protocol = request.lines().find_map(|line| {
14046            let (name, value) = line.split_once(':')?;
14047            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
14048                .then(|| value.trim().to_string())
14049        });
14050        let control_protocol = request.lines().find_map(|line| {
14051            let (name, value) = line.split_once(':')?;
14052            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
14053                .then(|| value.trim().to_string())
14054        });
14055        let request_number = {
14056            let mut requests = requests.lock().expect("captured requests");
14057            requests.push(CapturedRequest {
14058                method: method.to_string(),
14059                path: path.to_string(),
14060                authorization,
14061                namespace,
14062                worker_protocol: worker_protocol.clone(),
14063                control_protocol,
14064                body: body.to_string(),
14065                received_at: Instant::now(),
14066            });
14067            requests
14068                .iter()
14069                .filter(|request| request.path == path)
14070                .count()
14071        };
14072
14073        if behavior.reject_registration && path == "/api/worker/register" {
14074            write_mock_response(
14075                stream,
14076                "503 Service Unavailable",
14077                r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
14078            );
14079            return;
14080        }
14081
14082        if path.starts_with("/api/worker/registrations/") {
14083            if behavior.reject_deregistration_protocol {
14084                write_mock_response(
14085                    stream,
14086                    "400 Bad Request",
14087                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.1","requested_version":"1.2"}"#,
14088                );
14089            } else if behavior.reject_deregistration {
14090                write_mock_response(
14091                    stream,
14092                    "403 Forbidden",
14093                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
14094                );
14095            } else {
14096                write_mock_response(
14097                    stream,
14098                    "200 OK",
14099                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
14100                );
14101            }
14102            return;
14103        }
14104
14105        let is_poll = matches!(
14106            path,
14107            "/api/worker/workflow-tasks/poll"
14108                | "/api/worker/activity-tasks/poll"
14109                | "/api/worker/query-tasks/poll"
14110        );
14111        if is_poll && request_number <= behavior.poll_failures_per_path {
14112            return;
14113        }
14114        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
14115            return;
14116        }
14117        if path == "/api/worker/heartbeat"
14118            && behavior.heartbeat_failure_request == Some(request_number)
14119        {
14120            return;
14121        }
14122        if path == "/api/worker/heartbeat"
14123            && behavior.delayed_heartbeat_request == Some(request_number)
14124        {
14125            thread::sleep(behavior.heartbeat_response_delay);
14126        }
14127        if behavior.unauthorized_polls && is_poll {
14128            write_mock_response(
14129                stream,
14130                "401 Unauthorized",
14131                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
14132            );
14133            return;
14134        }
14135        if behavior.draining_polls && is_poll {
14136            write_mock_response(
14137                stream,
14138                "409 Conflict",
14139                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
14140            );
14141            return;
14142        }
14143
14144        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
14145            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
14146            let body = format!(
14147                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
14148            );
14149            write_mock_response(stream, "400 Bad Request", &body);
14150            return;
14151        }
14152
14153        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
14154        {
14155            write_mock_response(
14156                stream,
14157                "409 Conflict",
14158                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
14159            );
14160            return;
14161        }
14162
14163        if behavior.workflow_completion_status.is_some()
14164            && path == "/api/worker/workflow-tasks/poll"
14165            && request_number == 1
14166        {
14167            write_mock_response(
14168                stream,
14169                "200 OK",
14170                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
14171            );
14172            return;
14173        }
14174
14175        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
14176            if let (Some(status), Some(body)) = (
14177                behavior.workflow_completion_status,
14178                behavior.workflow_completion_body,
14179            ) {
14180                write_mock_response(stream, status, body);
14181                return;
14182            }
14183        }
14184
14185        if behavior.waiting_query_worker {
14186            if behavior.complete_named_signal
14187                && path == "/api/worker/workflow-tasks/poll"
14188                && request_number == 1
14189            {
14190                let body = json!({
14191                    "task": {
14192                        "task_id": "snapshot-open",
14193                        "workflow_id": "snapshot-1",
14194                        "run_id": "snapshot-run-1",
14195                        "workflow_type": "snapshot",
14196                        "payload_codec": DEFAULT_CODEC,
14197                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14198                            .expect("Avro workflow arguments"),
14199                        "history_events": [],
14200                        "workflow_task_attempt": 1,
14201                        "lease_owner": "rust-snapshot-worker"
14202                    }
14203                })
14204                .to_string();
14205                write_mock_response(stream, "200 OK", &body);
14206                return;
14207            }
14208
14209            let signal_request = request_number - usize::from(behavior.complete_named_signal);
14210            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
14211            if path == "/api/worker/workflow-tasks/poll"
14212                && signal_request >= 1
14213                && signal_request <= signal_request_limit
14214            {
14215                let finish = behavior.complete_named_signal && signal_request == 3;
14216                let amounts = if signal_request == 1 {
14217                    vec![3]
14218                } else {
14219                    vec![3, 5]
14220                };
14221                let task_id = if signal_request == 1 {
14222                    "snapshot-wait-3"
14223                } else if finish {
14224                    "snapshot-finish"
14225                } else {
14226                    "snapshot-wait-5"
14227                };
14228                let mut history_events = std::iter::once(json!({
14229                    "event_type": "SignalWaitOpened",
14230                    "payload": {"sequence": 1, "signal_name": "finish"}
14231                }))
14232                .chain(amounts.iter().enumerate().map(|(index, amount)| {
14233                    json!({
14234                        "event_type": "SignalReceived",
14235                        "payload": {
14236                            "signal_id": format!("increment-{amount}"),
14237                            "signal_name": "increment",
14238                            "workflow_sequence": index + 2,
14239                            "payload_codec": DEFAULT_CODEC,
14240                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14241                                .expect("Avro signal envelope")
14242                        }
14243                    })
14244                }))
14245                .collect::<Vec<_>>();
14246                let (resume_id, resume_name, resume_arguments) = if finish {
14247                    history_events.push(json!({
14248                        "event_type": "SignalReceived",
14249                        "payload": {
14250                            "signal_id": "finish",
14251                            "signal_name": "finish",
14252                            "workflow_sequence": 4,
14253                            "payload_codec": DEFAULT_CODEC,
14254                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14255                                .expect("Avro finish signal envelope")
14256                        }
14257                    }));
14258                    (
14259                        "finish".to_string(),
14260                        "finish".to_string(),
14261                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
14262                            .expect("Avro finish resume signal"),
14263                    )
14264                } else {
14265                    let amount = amounts.last().expect("amount");
14266                    (
14267                        format!("increment-{amount}"),
14268                        "increment".to_string(),
14269                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14270                            .expect("Avro increment resume signal"),
14271                    )
14272                };
14273                let body = json!({
14274                    "task": {
14275                        "task_id": task_id,
14276                        "workflow_id": "snapshot-1",
14277                        "run_id": "snapshot-run-1",
14278                        "workflow_type": "snapshot",
14279                        "payload_codec": DEFAULT_CODEC,
14280                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14281                            .expect("Avro workflow arguments"),
14282                        "history_events": history_events,
14283                        "workflow_task_attempt": 1,
14284                        "workflow_signal_id": resume_id,
14285                        "signal_name": resume_name,
14286                        "signal_arguments": resume_arguments,
14287                        "lease_owner": "rust-snapshot-worker"
14288                    }
14289                })
14290                .to_string();
14291                write_mock_response(stream, "200 OK", &body);
14292                return;
14293            }
14294
14295            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
14296                let history_events = [3, 5]
14297                    .into_iter()
14298                    .enumerate()
14299                    .map(|(index, amount)| {
14300                        json!({
14301                            "event_type": "SignalReceived",
14302                            "payload": {
14303                                "signal_id": format!("increment-{amount}"),
14304                                "signal_name": "increment",
14305                                "workflow_sequence": index + 2,
14306                                "payload_codec": DEFAULT_CODEC,
14307                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
14308                                    .expect("Avro query signal envelope")
14309                            }
14310                        })
14311                    })
14312                    .collect::<Vec<_>>();
14313                let body = json!({
14314                    "task": {
14315                        "query_task_id": "snapshot-current",
14316                        "query_task_attempt": 1,
14317                        "lease_owner": "rust-snapshot-worker",
14318                        "workflow_id": "snapshot-1",
14319                        "run_id": "snapshot-run-1",
14320                        "workflow_type": "snapshot",
14321                        "query_name": "current",
14322                        "payload_codec": DEFAULT_CODEC,
14323                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14324                            .expect("Avro workflow arguments"),
14325                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
14326                            .expect("Avro query arguments"),
14327                        "history_events": history_events,
14328                        "run_status": "waiting"
14329                    }
14330                })
14331                .to_string();
14332                write_mock_response(stream, "200 OK", &body);
14333                return;
14334            }
14335
14336            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
14337                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
14338            {
14339                write_mock_response(
14340                    stream,
14341                    "200 OK",
14342                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
14343                );
14344                return;
14345            }
14346
14347            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
14348                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
14349                return;
14350            }
14351
14352            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
14353                write_mock_response(
14354                    stream,
14355                    "200 OK",
14356                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
14357                );
14358                return;
14359            }
14360
14361            if path == "/api/worker/query-tasks/snapshot-current/complete" {
14362                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
14363                return;
14364            }
14365        }
14366
14367        if matches!(
14368            path,
14369            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
14370        ) {
14371            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14372                .expect("typed mock result");
14373            let body = json!({
14374                "result": typed_fidelity_probe().into_json().expect("result projection"),
14375                "result_envelope": result,
14376            })
14377            .to_string();
14378            write_mock_response(stream, "200 OK", &body);
14379            return;
14380        }
14381
14382        if path == "/api/workflows/typed-1" {
14383            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
14384                .expect("typed mock result");
14385            let body = json!({
14386                "workflow_id": "typed-1",
14387                "run_id": "run-typed-1",
14388                "workflow_type": "typed.echo",
14389                "status": "completed",
14390                "output": typed_fidelity_probe().into_json().expect("output projection"),
14391                "output_envelope": result,
14392            })
14393            .to_string();
14394            write_mock_response(stream, "200 OK", &body);
14395            return;
14396        }
14397
14398        let (status, body) = match path {
14399            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
14400            "/api/workflows" => (
14401                "201 Created",
14402                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
14403            ),
14404            "/api/worker/register" if behavior.decline_registration => (
14405                "200 OK",
14406                r#"{"worker_id":"declined-worker","registered":false}"#,
14407            ),
14408            "/api/worker/register" if behavior.waiting_query_worker => (
14409                "200 OK",
14410                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
14411            ),
14412            "/api/worker/register" => (
14413                "200 OK",
14414                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
14415            ),
14416            "/api/worker/heartbeat" => ("200 OK", "{}"),
14417            "/api/worker/activity-tasks/poll"
14418                if behavior.cancelled_activity && request_number == 1 =>
14419            {
14420                (
14421                    "200 OK",
14422                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"json","arguments":{"codec":"json","blob":"[]"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
14423                )
14424            }
14425            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
14426                ("200 OK", r#"{"task":null}"#)
14427            }
14428            "/api/worker/query-tasks/poll"
14429                if behavior.reject_query_completion && request_number == 1 =>
14430            {
14431                (
14432                    "200 OK",
14433                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"json","workflow_arguments":{"codec":"json","blob":"[]"},"query_arguments":{"codec":"json","blob":"[]"},"history_events":[],"run_status":"running"}}"#,
14434                )
14435            }
14436            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
14437            "/api/worker/query-tasks/query-capture/complete"
14438            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
14439            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
14440                "200 OK",
14441                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
14442            ),
14443            "/api/worker/activity-tasks/activity-cancel/complete" => (
14444                "409 Conflict",
14445                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
14446            ),
14447            "/api/worker/activity-tasks/activity-typed/complete"
14448            | "/api/worker/activity-tasks/activity-typed/fail"
14449            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
14450            "/api/workflows/counter-1/query/current" => (
14451                "200 OK",
14452                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"json","blob":"{\"count\":8}"}}"#,
14453            ),
14454            "/api/workflows/counter-1/query/missing" => (
14455                "404 Not Found",
14456                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
14457            ),
14458            "/api/workflows/wf-lifecycle/cancel" => (
14459                "200 OK",
14460                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
14461            ),
14462            "/api/workflows/wf-lifecycle/terminate" => (
14463                "200 OK",
14464                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
14465            ),
14466            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
14467                "200 OK",
14468                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
14469            ),
14470            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
14471                "200 OK",
14472                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
14473            ),
14474            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
14475            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
14476                "409 Conflict",
14477                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
14478            ),
14479            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
14480                "200 OK",
14481                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
14482            ),
14483            "/api/workflows/wf-cancelled" => (
14484                "200 OK",
14485                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
14486            ),
14487            "/api/workflows/wf-terminated" => (
14488                "200 OK",
14489                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
14490            ),
14491            "/api/workflows/wf-timed-out" => (
14492                "200 OK",
14493                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
14494            ),
14495            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
14496                "200 OK",
14497                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
14498            ),
14499            "/api/workflows/wf-selected" => (
14500                "200 OK",
14501                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
14502            ),
14503            "/api/workflows/wf-selected/runs/run-selected" => (
14504                "200 OK",
14505                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
14506            ),
14507            _ => ("404 Not Found", r#"{"message":"not found"}"#),
14508        };
14509        write_mock_response(stream, status, body);
14510    }
14511
14512    fn mock_request_is_complete(request: &[u8]) -> bool {
14513        let Some(header_end) = request
14514            .windows(4)
14515            .position(|window| window == b"\r\n\r\n")
14516            .map(|position| position + 4)
14517        else {
14518            return false;
14519        };
14520        let headers = String::from_utf8_lossy(&request[..header_end]);
14521        let content_length = headers.lines().find_map(|line| {
14522            let (name, value) = line.split_once(':')?;
14523            name.eq_ignore_ascii_case("content-length")
14524                .then(|| value.trim().parse::<usize>().ok())
14525                .flatten()
14526        });
14527
14528        request.len() >= header_end + content_length.unwrap_or(0)
14529    }
14530
14531    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
14532        let response = format!(
14533            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
14534            body.len()
14535        );
14536
14537        let _ = stream.write_all(response.as_bytes());
14538        let _ = stream.flush();
14539    }
14540}