Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::{
4    any::{type_name, Any, TypeId},
5    collections::{BTreeMap, HashMap},
6    future::Future,
7    io::{self, Read},
8    pin::Pin,
9    sync::{
10        atomic::{AtomicBool, Ordering},
11        Arc, Mutex, OnceLock,
12    },
13    task::{Context as TaskContext, Poll},
14    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
15};
16
17use apache_avro::{from_avro_datum, types::Value as AvroDatum, Schema};
18use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
19use chrono::DateTime;
20use futures_util::{future::OptionFuture, task::noop_waker_ref};
21use serde::{
22    de::DeserializeOwned,
23    ser::{SerializeMap, SerializeSeq},
24    Deserialize, Deserializer, Serialize, Serializer,
25};
26pub use serde_json::{json, Value};
27use sha2::{Digest, Sha256};
28use thiserror::Error;
29pub use uuid::Uuid;
30
31pub const WORKER_PROTOCOL_VERSION: &str = "1.16";
32pub const CONTROL_PLANE_VERSION: &str = "2";
33pub const DEFAULT_CODEC: &str = "avro";
34pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
35/// Worker-registration capability for server-routed read-only queries.
36pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
37/// Worker-registration capability for synchronous workflow updates.
38pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
39/// First additive worker protocol that defines query-task transport.
40pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
41/// First additive worker protocol that defines typed search-attribute upserts.
42pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
43/// First additive worker protocol that defines external durable condition waits.
44pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
45
46const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
47const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
48    "Workflow task waiting for scheduled history.";
49const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
50const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
51const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
52const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
53const MAX_MEMO_ENTRIES: usize = 100;
54const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
55const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
56
57const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
58    "lease_expired",
59    "query_task_not_found",
60    "query_task_not_leased",
61    "query_task_timed_out",
62];
63
64/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
65pub const AVRO_VALUE_SCHEMA_JSON: &str =
66    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
67pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
68pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
69const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
70
71static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
72
73#[derive(Clone, Copy)]
74enum RequestProtocol {
75    ControlPlane,
76    Worker(&'static str),
77}
78
79pub type Result<T> = std::result::Result<T, Error>;
80
81#[derive(Debug, Error)]
82pub enum Error {
83    #[error("transport error: {0}")]
84    Transport(#[from] reqwest::Error),
85    #[error(
86        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
87    )]
88    InvalidBaseUrl,
89    #[error("json error: {0}")]
90    Json(#[from] serde_json::Error),
91    #[error("http {status}: {body}")]
92    Http {
93        status: reqwest::StatusCode,
94        body: String,
95    },
96    #[error("codec error: {0}")]
97    Codec(String),
98    #[error(transparent)]
99    QueryFailed(QueryFailure),
100    #[error(transparent)]
101    Protocol(ProtocolFailure),
102    #[error(transparent)]
103    NonDeterministicReplay(ReplayFailure),
104    #[error(transparent)]
105    ChildWorkflowFailed(ChildWorkflowFailure),
106    #[error(transparent)]
107    ActivityFailed(ActivityFailure),
108    #[error(transparent)]
109    ParallelFailed(ParallelFailure),
110    #[error(transparent)]
111    SagaCompensationFailed(SagaCompensationFailure),
112    #[error(transparent)]
113    InvalidParallelGroup(ParallelGroupError),
114    #[error(transparent)]
115    WorkflowCancellationRequested(WorkflowCancellationRequested),
116    #[error(transparent)]
117    WorkflowCommandRejected(WorkflowCommandRejection),
118    #[error(transparent)]
119    WorkflowFailed(WorkflowTerminalOutcome),
120    #[error(transparent)]
121    WorkflowCancelled(WorkflowTerminalOutcome),
122    #[error(transparent)]
123    WorkflowTerminated(WorkflowTerminalOutcome),
124    #[error(transparent)]
125    WorkflowTimedOut(WorkflowTerminalOutcome),
126    #[error(transparent)]
127    ActivityTaskRejected(ActivityTaskRejection),
128    #[error("workflow handler {0:?} is not registered")]
129    WorkflowNotRegistered(String),
130    #[error("activity handler {0:?} is not registered")]
131    ActivityNotRegistered(String),
132    #[error(
133        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
134    )]
135    HandlerType {
136        handler_kind: HandlerKind,
137        handler_name: String,
138        value_kind: HandlerValueKind,
139        rust_type: &'static str,
140        message: String,
141    },
142    #[error("workflow future yielded without emitting a durable command")]
143    WorkflowYieldedWithoutCommand,
144    #[error(
145        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
146    )]
147    MissingWorkflowCommandIdentity,
148    #[error("workflow state lock is poisoned")]
149    WorkflowStatePoisoned,
150    #[error("timer duration is too large for the worker protocol")]
151    TimerDurationOverflow,
152    #[error(transparent)]
153    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
154    #[error(transparent)]
155    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
156    #[error("operation timed out")]
157    Timeout,
158    #[error(
159        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
160    )]
161    MissingRoleCredentials {
162        role: &'static str,
163        opposite_role: &'static str,
164    },
165    #[error("worker loop error: {0}")]
166    WorkerLoop(String),
167    #[error(
168        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
169    )]
170    UnsupportedUpdateValidators { workflow_type: String },
171    #[error("{primary}; worker deregistration also failed: {deregistration}")]
172    WorkerShutdown {
173        primary: Box<Error>,
174        deregistration: Box<Error>,
175    },
176    #[error("invalid child workflow options: {0}")]
177    InvalidChildWorkflowOptions(String),
178    #[error("invalid workflow memo update: {0}")]
179    InvalidMemoUpdate(String),
180    #[error(
181        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
182    )]
183    WorkflowMemoUpdatesUnavailable,
184    #[error(transparent)]
185    InvalidActivityOptions(ActivityOptionsError),
186    #[error(transparent)]
187    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
188    #[doc(hidden)]
189    #[error("workflow requested continue as new")]
190    ContinueAsNew(ContinueAsNewRequest),
191}
192
193/// Validation failure for a durable condition-wait definition.
194#[derive(Clone, Debug, Error, PartialEq, Eq)]
195pub enum ConditionWaitOptionsError {
196    #[error("condition_key must be non-empty")]
197    EmptyKey,
198    #[error("condition_definition_fingerprint must be non-empty")]
199    EmptyPredicateIdentity,
200    #[error("condition timeout is too large for the worker protocol")]
201    TimeoutOverflow,
202}
203
204/// Stable identity and optional durable timeout for a condition wait.
205///
206/// `predicate_identity` is recorded as the worker protocol's
207/// `condition_definition_fingerprint` and must change whenever predicate
208/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
209/// is written inline; it derives this identity from the predicate tokens.
210#[derive(Clone, Debug, PartialEq, Eq)]
211pub struct ConditionWaitOptions {
212    condition_key: String,
213    predicate_identity: String,
214    timeout: Option<Duration>,
215}
216
217impl ConditionWaitOptions {
218    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
219        Self {
220            condition_key: condition_key.into(),
221            predicate_identity: predicate_identity.into(),
222            timeout: None,
223        }
224    }
225
226    pub fn timeout(mut self, timeout: Duration) -> Self {
227        self.timeout = Some(timeout);
228        self
229    }
230
231    fn validate(
232        &self,
233    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
234        let condition_key = self.condition_key.trim();
235        if condition_key.is_empty() {
236            return Err(ConditionWaitOptionsError::EmptyKey);
237        }
238        let predicate_identity = self.predicate_identity.trim();
239        if predicate_identity.is_empty() {
240            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
241        }
242        let timeout_seconds = self
243            .timeout
244            .map(|timeout| {
245                timeout
246                    .as_secs()
247                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
248                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
249            })
250            .transpose()?;
251
252        Ok(ValidatedConditionWaitOptions {
253            condition_key: condition_key.to_string(),
254            predicate_identity: predicate_identity.to_string(),
255            timeout_seconds,
256        })
257    }
258}
259
260#[derive(Clone, Debug, PartialEq, Eq)]
261struct ValidatedConditionWaitOptions {
262    condition_key: String,
263    predicate_identity: String,
264    timeout_seconds: Option<u64>,
265}
266
267/// Unambiguous terminal result of a durable condition wait.
268#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
269#[serde(rename_all = "snake_case")]
270pub enum ConditionWaitResult {
271    Satisfied,
272    TimedOut,
273}
274
275impl ConditionWaitResult {
276    pub fn is_satisfied(self) -> bool {
277        self == Self::Satisfied
278    }
279
280    pub fn is_timed_out(self) -> bool {
281        self == Self::TimedOut
282    }
283}
284
285/// Build the stable condition definition identity used by [`wait_condition!`].
286#[doc(hidden)]
287pub fn __condition_definition_fingerprint(source: &str) -> String {
288    let mut digest = Sha256::new();
289    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
290    digest.update(source.as_bytes());
291    format!("sha256:{:x}", digest.finalize())
292}
293
294/// Create a durable condition wait whose predicate definition is fingerprinted
295/// from its inline Rust tokens.
296///
297/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
298/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
299#[macro_export]
300macro_rules! wait_condition {
301    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
302        $ctx.wait_condition(
303            $crate::ConditionWaitOptions::new(
304                $key,
305                $crate::__condition_definition_fingerprint(concat!(
306                    module_path!(),
307                    "\0",
308                    stringify!($predicate)
309                )),
310            )
311            .timeout($timeout),
312            $predicate,
313        )
314    }};
315    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
316        $ctx.wait_condition(
317            $crate::ConditionWaitOptions::new(
318                $key,
319                $crate::__condition_definition_fingerprint(concat!(
320                    module_path!(),
321                    "\0",
322                    stringify!($predicate)
323                )),
324            ),
325            $predicate,
326        )
327    }};
328}
329
330const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
331const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
332const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 255;
333const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
334const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
335
336/// Validation failure for a typed workflow search-attribute update.
337#[derive(Clone, Debug, Error, PartialEq, Eq)]
338pub enum SearchAttributeUpdateError {
339    #[error("search-attribute update requires at least one attribute")]
340    Empty,
341    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
342    InvalidKey(String),
343    #[error("search-attribute update exceeds the limit of 100 attributes")]
344    TooManyAttributes,
345    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
346    ValueTooLong {
347        key: String,
348        kind: &'static str,
349        limit: usize,
350    },
351    #[error(
352        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
353    )]
354    EmptyString(String),
355    #[error("search attribute {0:?} has a non-finite float value")]
356    NonFiniteFloat(String),
357    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
358    InvalidDateTime(String),
359    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
360    PayloadTooLarge,
361}
362
363/// One public typed search-attribute value.
364#[derive(Clone, Debug, PartialEq)]
365pub enum SearchAttributeValue {
366    String(String),
367    Keyword(String),
368    KeywordList(Vec<String>),
369    Int(i64),
370    Float(f64),
371    Bool(bool),
372    DateTime(String),
373    Delete,
374}
375
376impl SearchAttributeValue {
377    fn type_name(&self) -> Option<&'static str> {
378        match self {
379            Self::String(_) => Some("string"),
380            Self::Keyword(_) => Some("keyword"),
381            Self::KeywordList(_) => Some("keyword_list"),
382            Self::Int(_) => Some("int"),
383            Self::Float(_) => Some("float"),
384            Self::Bool(_) => Some("bool"),
385            Self::DateTime(_) => Some("datetime"),
386            Self::Delete => None,
387        }
388    }
389
390    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
391        let normalize_string = |value: String, kind: &'static str, limit: usize| {
392            let value = value.trim().to_string();
393            if value.is_empty() {
394                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
395            }
396            if value.len() > limit {
397                return Err(SearchAttributeUpdateError::ValueTooLong {
398                    key: key.to_string(),
399                    kind,
400                    limit,
401                });
402            }
403            Ok(value)
404        };
405
406        match self {
407            Self::String(value) => Ok(Self::String(normalize_string(
408                value,
409                "string",
410                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
411            )?)),
412            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
413                value,
414                "keyword",
415                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
416            )?)),
417            Self::KeywordList(values) => {
418                let values = values
419                    .into_iter()
420                    .map(|value| {
421                        let value = value.trim().to_string();
422                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
423                            return Err(SearchAttributeUpdateError::ValueTooLong {
424                                key: key.to_string(),
425                                kind: "keyword-list entry",
426                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
427                            });
428                        }
429                        Ok(value)
430                    })
431                    .collect::<std::result::Result<Vec<_>, _>>()?;
432                Ok(Self::KeywordList(values))
433            }
434            Self::Float(value) if !value.is_finite() => {
435                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
436            }
437            Self::DateTime(value) => {
438                let value =
439                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
440                if DateTime::parse_from_rfc3339(&value).is_err() {
441                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
442                }
443                Ok(Self::DateTime(value))
444            }
445            value => Ok(value),
446        }
447    }
448
449    fn into_json(self) -> Value {
450        match self {
451            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
452                Value::String(value)
453            }
454            Self::KeywordList(values) => {
455                Value::Array(values.into_iter().map(Value::String).collect())
456            }
457            Self::Int(value) => json!(value),
458            Self::Float(value) => json!(value),
459            Self::Bool(value) => json!(value),
460            Self::Delete => Value::Null,
461        }
462    }
463}
464
465/// Validated typed workflow-side search-attribute mutation.
466#[derive(Clone, Debug, Default, PartialEq)]
467pub struct SearchAttributeUpdate {
468    attributes: BTreeMap<String, SearchAttributeValue>,
469}
470
471impl SearchAttributeUpdate {
472    pub fn new() -> Self {
473        Self::default()
474    }
475
476    pub fn set(
477        mut self,
478        key: impl Into<String>,
479        value: SearchAttributeValue,
480    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
481        let key = key.into();
482        validate_search_attribute_key(&key)?;
483        if !self.attributes.contains_key(&key)
484            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
485        {
486            return Err(SearchAttributeUpdateError::TooManyAttributes);
487        }
488        self.attributes.insert(key.clone(), value.normalized(&key)?);
489        self.validate_size()?;
490        Ok(self)
491    }
492
493    pub fn string(
494        self,
495        key: impl Into<String>,
496        value: impl Into<String>,
497    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
498        self.set(key, SearchAttributeValue::String(value.into()))
499    }
500
501    pub fn keyword(
502        self,
503        key: impl Into<String>,
504        value: impl Into<String>,
505    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
506        self.set(key, SearchAttributeValue::Keyword(value.into()))
507    }
508
509    pub fn keyword_list<I, V>(
510        self,
511        key: impl Into<String>,
512        values: I,
513    ) -> std::result::Result<Self, SearchAttributeUpdateError>
514    where
515        I: IntoIterator<Item = V>,
516        V: Into<String>,
517    {
518        self.set(
519            key,
520            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
521        )
522    }
523
524    pub fn int(
525        self,
526        key: impl Into<String>,
527        value: i64,
528    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
529        self.set(key, SearchAttributeValue::Int(value))
530    }
531
532    pub fn float(
533        self,
534        key: impl Into<String>,
535        value: f64,
536    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
537        self.set(key, SearchAttributeValue::Float(value))
538    }
539
540    pub fn bool(
541        self,
542        key: impl Into<String>,
543        value: bool,
544    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
545        self.set(key, SearchAttributeValue::Bool(value))
546    }
547
548    pub fn datetime(
549        self,
550        key: impl Into<String>,
551        value: impl Into<String>,
552    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553        self.set(key, SearchAttributeValue::DateTime(value.into()))
554    }
555
556    pub fn delete(
557        self,
558        key: impl Into<String>,
559    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
560        self.set(key, SearchAttributeValue::Delete)
561    }
562
563    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
564        let (attributes, _) = self.clone().into_wire_parts();
565        if serde_json::to_vec(&attributes)
566            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
567            .unwrap_or(true)
568        {
569            return Err(SearchAttributeUpdateError::PayloadTooLarge);
570        }
571        Ok(())
572    }
573
574    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
575        let mut attributes = serde_json::Map::new();
576        let mut attribute_types = BTreeMap::new();
577        for (key, value) in self.attributes {
578            if let Some(type_name) = value.type_name() {
579                attribute_types.insert(key.clone(), type_name.to_string());
580            }
581            attributes.insert(key, value.into_json());
582        }
583        (Value::Object(attributes), attribute_types)
584    }
585
586    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
587        if self.attributes.is_empty() {
588            return Err(SearchAttributeUpdateError::Empty);
589        }
590        self.validate_size()
591    }
592}
593
594fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
595    let valid = !key.is_empty()
596        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
597        && key
598            .bytes()
599            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
600    if valid {
601        Ok(())
602    } else {
603        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
604    }
605}
606
607/// The registered handler family reported by [`Error::HandlerType`].
608#[derive(Clone, Copy, Debug, PartialEq, Eq)]
609pub enum HandlerKind {
610    Workflow,
611    Activity,
612}
613
614impl std::fmt::Display for HandlerKind {
615    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
616        formatter.write_str(match self {
617            Self::Workflow => "workflow",
618            Self::Activity => "activity",
619        })
620    }
621}
622
623/// Whether a typed handler failed to adapt its input or result.
624#[derive(Clone, Copy, Debug, PartialEq, Eq)]
625pub enum HandlerValueKind {
626    Input,
627    Result,
628}
629
630impl std::fmt::Display for HandlerValueKind {
631    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
632        formatter.write_str(match self {
633            Self::Input => "input",
634            Self::Result => "result",
635        })
636    }
637}
638
639/// The lifecycle command sent to a workflow execution.
640#[derive(Clone, Copy, Debug, PartialEq, Eq)]
641pub enum WorkflowCommandKind {
642    Cancel,
643    Terminate,
644}
645
646impl WorkflowCommandKind {
647    fn as_str(self) -> &'static str {
648        match self {
649            Self::Cancel => "cancel",
650            Self::Terminate => "terminate",
651        }
652    }
653}
654
655/// Optional structured fields for a cancellation or termination request.
656#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
657pub struct WorkflowCommandOptions {
658    #[serde(skip_serializing_if = "Option::is_none")]
659    pub reason: Option<String>,
660    #[serde(skip_serializing_if = "Option::is_none")]
661    pub request_id: Option<String>,
662}
663
664/// Server-enforced timeout policy for a workflow start.
665///
666/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
667/// only bounds how long the caller waits. A server deadline produces a terminal
668/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
669/// `run_timeout`.
670#[derive(Clone, Debug, PartialEq, Eq)]
671pub struct WorkflowStartOptions {
672    pub execution_timeout_seconds: u64,
673    pub run_timeout_seconds: u64,
674}
675
676impl Default for WorkflowStartOptions {
677    fn default() -> Self {
678        Self {
679            execution_timeout_seconds: 3600,
680            run_timeout_seconds: 600,
681        }
682    }
683}
684
685impl WorkflowStartOptions {
686    pub fn new() -> Self {
687        Self::default()
688    }
689
690    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
691        self.execution_timeout_seconds = seconds;
692        self
693    }
694
695    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
696        self.run_timeout_seconds = seconds;
697        self
698    }
699
700    fn validate(&self) -> Result<()> {
701        if self.execution_timeout_seconds == 0 {
702            return Err(Error::Codec(
703                "execution_timeout_seconds must be at least 1".to_string(),
704            ));
705        }
706        if self.run_timeout_seconds == 0 {
707            return Err(Error::Codec(
708                "run_timeout_seconds must be at least 1".to_string(),
709            ));
710        }
711        if self.run_timeout_seconds > self.execution_timeout_seconds {
712            return Err(Error::Codec(
713                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
714            ));
715        }
716
717        Ok(())
718    }
719}
720
721/// Optional routing overrides for a continue-as-new transition.
722///
723/// Omitted values retain the current workflow type and task queue. Server-owned
724/// instance metadata is not accepted here and is carried by the server.
725#[derive(Clone, Debug, Default, PartialEq, Eq)]
726pub struct ContinueAsNewOptions {
727    pub workflow_type: Option<String>,
728    pub task_queue: Option<String>,
729}
730
731impl ContinueAsNewOptions {
732    pub fn new() -> Self {
733        Self::default()
734    }
735
736    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
737        self.workflow_type = Some(workflow_type.into());
738        self
739    }
740
741    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
742        self.task_queue = Some(task_queue.into());
743        self
744    }
745
746    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
747        for (field, value) in [
748            ("workflow_type", self.workflow_type.as_deref()),
749            ("task_queue", self.task_queue.as_deref()),
750        ] {
751            if value.is_some_and(|value| value.trim().is_empty()) {
752                return Err(ContinueAsNewOptionsError {
753                    field,
754                    message: format!("{field} must not be empty"),
755                });
756            }
757        }
758        Ok(())
759    }
760}
761
762/// A stable validation error raised before a continue-as-new command is emitted.
763#[derive(Clone, Debug, Error, PartialEq, Eq)]
764#[error("invalid continue-as-new option {field}: {message}")]
765pub struct ContinueAsNewOptionsError {
766    pub field: &'static str,
767    pub message: String,
768}
769
770/// Public history-budget information attached to the current workflow task.
771#[derive(Clone, Debug, Default, PartialEq, Eq)]
772pub struct WorkflowHistoryBudget {
773    pub event_count: u64,
774    pub size_bytes: Option<u64>,
775    pub continue_as_new_recommended: bool,
776    pub pressure: Option<String>,
777}
778
779#[doc(hidden)]
780#[derive(Clone, Debug)]
781pub struct ContinueAsNewRequest {
782    arguments: AvroValue,
783    options: ContinueAsNewOptions,
784}
785
786impl WorkflowCommandOptions {
787    pub fn new() -> Self {
788        Self::default()
789    }
790
791    pub fn reason(mut self, reason: impl Into<String>) -> Self {
792        self.reason = Some(reason.into());
793        self
794    }
795
796    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
797        self.request_id = Some(request_id.into());
798        self
799    }
800}
801
802/// The accepted, machine-readable result of a lifecycle command.
803#[derive(Clone, Debug, PartialEq)]
804pub struct WorkflowCommandResult {
805    pub command: WorkflowCommandKind,
806    pub workflow_id: String,
807    pub run_id: Option<String>,
808    pub outcome: Option<String>,
809    pub reason: Option<String>,
810    pub command_status: Option<String>,
811    pub raw: Value,
812}
813
814/// A stable rejection returned by instance- or selected-run lifecycle commands.
815#[derive(Clone, Debug, Error)]
816#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
817pub struct WorkflowCommandRejection {
818    pub command: WorkflowCommandKind,
819    pub status: u16,
820    pub reason: String,
821    pub message: String,
822    pub workflow_id: String,
823    pub run_id: Option<String>,
824    pub target_scope: Option<String>,
825    pub body: Value,
826}
827
828/// Stable terminal categories returned by [`WorkflowHandle::result`].
829#[derive(Clone, Copy, Debug, PartialEq, Eq)]
830pub enum WorkflowTerminalKind {
831    Failed,
832    Cancelled,
833    Terminated,
834    TimedOut,
835}
836
837/// A typed terminal workflow outcome with durable identity and failure metadata.
838///
839/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
840/// of parsing its display representation. Fields remain `None` when an older
841/// server did not publish that metadata.
842#[derive(Clone, Debug, Error)]
843#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
844pub struct WorkflowTerminalOutcome {
845    pub kind: WorkflowTerminalKind,
846    pub workflow_id: String,
847    pub run_id: Option<String>,
848    pub reason: String,
849    pub failure_category: Option<String>,
850    pub failure_id: Option<String>,
851    pub exception_type: Option<String>,
852    pub exception_class: Option<String>,
853    pub non_retryable: Option<bool>,
854    pub message: Option<String>,
855    pub exception: Option<Value>,
856    pub raw: Value,
857}
858
859/// A worker-side activity settlement or heartbeat rejected by durable state.
860#[derive(Clone, Debug, Error)]
861#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
862pub struct ActivityTaskRejection {
863    pub operation: String,
864    pub status: u16,
865    pub reason: String,
866    pub task_id: String,
867    pub activity_attempt_id: String,
868    pub cancel_requested: bool,
869    pub can_continue: Option<bool>,
870    pub run_closed_reason: Option<String>,
871    pub body: Value,
872}
873
874/// Stable validation categories for [`ActivityOptions`].
875#[derive(Clone, Copy, Debug, PartialEq, Eq)]
876pub enum ActivityOptionsErrorKind {
877    EmptyTaskQueue,
878    EmptyRetryPolicy,
879    InvalidMaxAttempts,
880    BackoffWithoutRetryBudget,
881    TooManyBackoffIntervals,
882    InvalidBackoffCoefficient,
883    BackoffGenerationTooLarge,
884    BackoffOverflow,
885    EmptyNonRetryableErrorType,
886    TimeoutNotPositive,
887    TimeoutOverflow,
888    TimeoutOrder,
889}
890
891/// A machine-readable activity-options validation failure.
892#[derive(Clone, Debug, Error, PartialEq, Eq)]
893#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
894pub struct ActivityOptionsError {
895    pub kind: ActivityOptionsErrorKind,
896    pub field: Option<&'static str>,
897    pub message: String,
898}
899
900impl ActivityOptionsError {
901    fn new(
902        kind: ActivityOptionsErrorKind,
903        field: Option<&'static str>,
904        message: impl Into<String>,
905    ) -> Self {
906        Self {
907            kind,
908            field,
909            message: message.into(),
910        }
911    }
912}
913
914/// Stable terminal categories returned when an awaited activity does not succeed.
915#[derive(Clone, Copy, Debug, PartialEq, Eq)]
916pub enum ActivityFailureKind {
917    Failed,
918    Cancelled,
919    TimedOut,
920}
921
922/// A stable, machine-readable terminal activity failure.
923///
924/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
925/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
926#[derive(Clone, Debug, Error)]
927#[error("activity failed ({reason}): {message}")]
928pub struct ActivityFailure {
929    pub kind: ActivityFailureKind,
930    pub reason: String,
931    pub message: String,
932    pub activity_execution_id: Option<String>,
933    pub activity_attempt_id: Option<String>,
934    pub activity_type: Option<String>,
935    pub activity_class: Option<String>,
936    pub attempt_number: Option<u64>,
937    pub failure_id: Option<String>,
938    pub failure_category: Option<String>,
939    pub timeout_kind: Option<String>,
940    pub non_retryable: bool,
941    pub exception_type: Option<String>,
942    pub exception_class: Option<String>,
943    pub code: Option<Value>,
944    pub exception: Option<Value>,
945}
946
947/// Stable terminal categories returned when an awaited child does not succeed.
948#[derive(Clone, Copy, Debug, PartialEq, Eq)]
949pub enum ChildWorkflowFailureKind {
950    Failed,
951    Cancelled,
952    Terminated,
953}
954
955/// A stable, machine-readable child workflow failure delivered to its parent.
956///
957/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
958/// of parsing the display message. Child and parent identifiers retain the
959/// relationship recorded in durable history across worker restarts.
960#[derive(Clone, Debug, Error)]
961#[error("child workflow failed ({reason}): {message}")]
962pub struct ChildWorkflowFailure {
963    pub kind: ChildWorkflowFailureKind,
964    pub reason: String,
965    pub message: String,
966    pub parent_workflow_id: Option<String>,
967    pub parent_workflow_run_id: Option<String>,
968    pub child_workflow_id: Option<String>,
969    pub child_workflow_run_id: Option<String>,
970    pub child_workflow_type: Option<String>,
971    pub failure_id: Option<String>,
972    pub failure_category: Option<String>,
973    pub exception_type: Option<String>,
974    pub exception_class: Option<String>,
975    pub non_retryable: bool,
976    pub code: Option<Value>,
977    pub exception: Option<Value>,
978}
979
980/// The identity of one durable workflow execution.
981#[derive(Clone, Debug, PartialEq, Eq)]
982pub struct WorkflowIdentity {
983    pub workflow_id: Option<String>,
984    pub run_id: Option<String>,
985}
986
987/// A successful child result together with its durable parent-child identity.
988#[derive(Clone, Debug, PartialEq)]
989pub struct ChildWorkflowResult {
990    pub parent: WorkflowIdentity,
991    pub child: WorkflowIdentity,
992    pub child_workflow_type: Option<String>,
993    pub result: Value,
994}
995
996/// Lossless successful child result for fixed Avro Value workflows.
997#[derive(Clone, Debug, PartialEq)]
998pub struct ChildWorkflowAvroResult {
999    pub parent: WorkflowIdentity,
1000    pub child: WorkflowIdentity,
1001    pub child_workflow_type: Option<String>,
1002    pub result: AvroValue,
1003}
1004
1005/// Stable identity for one enclosing deterministic parallel group.
1006///
1007/// The same fields are attached to every ordinary activity, timer, or child
1008/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1009/// no Rust-specific wire command is introduced.
1010#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1011pub struct ParallelGroupMetadata {
1012    pub parallel_group_id: String,
1013    pub parallel_group_kind: String,
1014    pub parallel_group_base_sequence: u64,
1015    pub parallel_group_size: usize,
1016    pub parallel_group_index: usize,
1017}
1018
1019/// One input-ordered result returned by [`WorkflowContext::parallel`].
1020#[derive(Clone, Debug, PartialEq)]
1021pub enum ParallelResult {
1022    Activity(Value),
1023    ChildWorkflow(ChildWorkflowResult),
1024    Timer,
1025    Group(Vec<ParallelResult>),
1026}
1027
1028/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1029#[derive(Clone, Debug, PartialEq)]
1030pub enum ParallelAvroResult {
1031    Activity(AvroValue),
1032    ChildWorkflow(ChildWorkflowAvroResult),
1033    Timer,
1034    Group(Vec<ParallelAvroResult>),
1035}
1036
1037impl ParallelAvroResult {
1038    fn into_json_result(self) -> Result<ParallelResult> {
1039        match self {
1040            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1041            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1042                parent: result.parent,
1043                child: result.child,
1044                child_workflow_type: result.child_workflow_type,
1045                result: result.result.into_json()?,
1046            })),
1047            Self::Timer => Ok(ParallelResult::Timer),
1048            Self::Group(results) => Ok(ParallelResult::Group(
1049                results
1050                    .into_iter()
1051                    .map(Self::into_json_result)
1052                    .collect::<Result<Vec<_>>>()?,
1053            )),
1054        }
1055    }
1056}
1057
1058/// One successful leaf retained when another parallel member failed.
1059#[derive(Clone, Debug, PartialEq)]
1060pub struct ParallelCompletion {
1061    pub member_path: Vec<usize>,
1062    pub result: ParallelResult,
1063}
1064
1065/// A deterministic join failed after some siblings had already completed.
1066///
1067/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1068/// error. `completed` is declaration ordered and contains only durable
1069/// successes observed in the same replay. Late sibling completions can add
1070/// entries on a later replay without changing `member_path` or the selected
1071/// positional failure.
1072#[derive(Debug, Error)]
1073#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1074pub struct ParallelFailure {
1075    pub group_id: String,
1076    pub member_path: Vec<usize>,
1077    pub group_path: Vec<ParallelGroupMetadata>,
1078    pub completed: Vec<ParallelCompletion>,
1079    #[source]
1080    pub cause: Box<Error>,
1081}
1082
1083/// Stable validation error returned before an invalid group emits commands.
1084#[derive(Clone, Debug, Error, PartialEq, Eq)]
1085#[error("invalid deterministic parallel group ({reason}): {message}")]
1086pub struct ParallelGroupError {
1087    pub reason: &'static str,
1088    pub member_path: Vec<usize>,
1089    pub message: String,
1090}
1091
1092/// Cooperative workflow cancellation observed at an author-controlled point.
1093#[derive(Clone, Debug, Error, PartialEq, Eq)]
1094#[error("workflow cancellation was requested")]
1095pub struct WorkflowCancellationRequested;
1096
1097/// A forward saga failure followed by a terminal compensation failure.
1098#[derive(Debug, Error)]
1099#[error(
1100    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1101)]
1102pub struct SagaCompensationFailure {
1103    pub initiating_failure: Box<Error>,
1104    pub compensation_failure: Box<Error>,
1105    pub compensation_activity_type: String,
1106    pub compensation_registration_order: usize,
1107}
1108
1109/// Server behavior when a parent closes while its child is still open.
1110#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1111pub enum ParentClosePolicy {
1112    #[default]
1113    Abandon,
1114    RequestCancel,
1115    Terminate,
1116}
1117
1118impl ParentClosePolicy {
1119    fn as_str(self) -> &'static str {
1120        match self {
1121            Self::Abandon => "abandon",
1122            Self::RequestCancel => "request_cancel",
1123            Self::Terminate => "terminate",
1124        }
1125    }
1126}
1127
1128/// Durable retry policy for one child workflow invocation.
1129#[derive(Clone, Debug, Default, PartialEq, Eq)]
1130pub struct ChildWorkflowRetryPolicy {
1131    pub max_attempts: Option<u32>,
1132    pub backoff_seconds: Vec<u64>,
1133    pub non_retryable_error_types: Vec<String>,
1134}
1135
1136/// Options recorded with a child-workflow command.
1137///
1138/// The task queue is mandatory so routing is explicit and replay-stable.
1139#[derive(Clone, Debug, PartialEq, Eq)]
1140pub struct ChildWorkflowOptions {
1141    pub task_queue: String,
1142    pub parent_close_policy: ParentClosePolicy,
1143    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1144    pub execution_timeout_seconds: Option<u64>,
1145    pub run_timeout_seconds: Option<u64>,
1146}
1147
1148impl ChildWorkflowOptions {
1149    pub fn new(task_queue: impl Into<String>) -> Self {
1150        Self {
1151            task_queue: task_queue.into(),
1152            parent_close_policy: ParentClosePolicy::Abandon,
1153            retry_policy: None,
1154            execution_timeout_seconds: None,
1155            run_timeout_seconds: None,
1156        }
1157    }
1158
1159    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1160        self.parent_close_policy = policy;
1161        self
1162    }
1163
1164    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1165        self.retry_policy = Some(policy);
1166        self
1167    }
1168
1169    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1170        self.execution_timeout_seconds = Some(seconds);
1171        self
1172    }
1173
1174    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1175        self.run_timeout_seconds = Some(seconds);
1176        self
1177    }
1178}
1179
1180/// Backoff intervals for one durable activity retry policy.
1181#[derive(Clone, Debug, PartialEq, Eq)]
1182pub enum ActivityBackoff {
1183    /// Use these intervals between attempts. The server repeats the final
1184    /// interval if the retry budget contains more attempts than entries.
1185    Explicit(Vec<Duration>),
1186    /// Generate one interval for every retry using integer exponential growth.
1187    Exponential {
1188        initial_interval: Duration,
1189        coefficient: u32,
1190        maximum_interval: Option<Duration>,
1191    },
1192}
1193
1194/// Durable server-side retry policy for one activity execution.
1195#[derive(Clone, Debug, Default, PartialEq, Eq)]
1196pub struct ActivityRetryPolicy {
1197    pub max_attempts: Option<u32>,
1198    pub backoff: Option<ActivityBackoff>,
1199    pub non_retryable_error_types: Vec<String>,
1200}
1201
1202impl ActivityRetryPolicy {
1203    /// Start a policy with a finite attempt budget, including the first attempt.
1204    pub fn new(max_attempts: u32) -> Self {
1205        Self {
1206            max_attempts: Some(max_attempts),
1207            ..Self::default()
1208        }
1209    }
1210
1211    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1212        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1213        self
1214    }
1215
1216    pub fn exponential_backoff(
1217        mut self,
1218        initial_interval: Duration,
1219        coefficient: u32,
1220        maximum_interval: Option<Duration>,
1221    ) -> Self {
1222        self.backoff = Some(ActivityBackoff::Exponential {
1223            initial_interval,
1224            coefficient,
1225            maximum_interval,
1226        });
1227        self
1228    }
1229
1230    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1231        self.non_retryable_error_types.push(error_type.into());
1232        self
1233    }
1234
1235    pub fn non_retryable_error_types(
1236        mut self,
1237        error_types: impl IntoIterator<Item = impl Into<String>>,
1238    ) -> Self {
1239        self.non_retryable_error_types
1240            .extend(error_types.into_iter().map(Into::into));
1241        self
1242    }
1243}
1244
1245/// Options recorded atomically on one deterministic `schedule_activity` command.
1246///
1247/// Durations are rounded up to whole seconds when encoded, so the server never
1248/// receives a shorter timeout or backoff than the caller requested.
1249#[derive(Clone, Debug, Default, PartialEq, Eq)]
1250pub struct ActivityOptions {
1251    pub task_queue: Option<String>,
1252    pub retry_policy: Option<ActivityRetryPolicy>,
1253    pub start_to_close_timeout: Option<Duration>,
1254    pub schedule_to_start_timeout: Option<Duration>,
1255    pub schedule_to_close_timeout: Option<Duration>,
1256    pub heartbeat_timeout: Option<Duration>,
1257}
1258
1259impl ActivityOptions {
1260    pub fn new() -> Self {
1261        Self::default()
1262    }
1263
1264    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1265        self.task_queue = Some(task_queue.into());
1266        self
1267    }
1268
1269    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1270        self.retry_policy = Some(policy);
1271        self
1272    }
1273
1274    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1275        self.start_to_close_timeout = Some(timeout);
1276        self
1277    }
1278
1279    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1280        self.schedule_to_start_timeout = Some(timeout);
1281        self
1282    }
1283
1284    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1285        self.schedule_to_close_timeout = Some(timeout);
1286        self
1287    }
1288
1289    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1290        self.heartbeat_timeout = Some(timeout);
1291        self
1292    }
1293
1294    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1295        if self
1296            .task_queue
1297            .as_deref()
1298            .is_some_and(|queue| queue.trim().is_empty())
1299        {
1300            return Err(ActivityOptionsError::new(
1301                ActivityOptionsErrorKind::EmptyTaskQueue,
1302                Some("task_queue"),
1303                "task_queue must not be empty",
1304            ));
1305        }
1306
1307        for (field, value) in [
1308            ("start_to_close_timeout", self.start_to_close_timeout),
1309            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1310            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1311            ("heartbeat_timeout", self.heartbeat_timeout),
1312        ] {
1313            if value.is_some_and(|value| value.is_zero()) {
1314                return Err(ActivityOptionsError::new(
1315                    ActivityOptionsErrorKind::TimeoutNotPositive,
1316                    Some(field),
1317                    format!("{field} must be positive"),
1318                ));
1319            }
1320        }
1321
1322        validate_timeout_order(
1323            "heartbeat_timeout",
1324            self.heartbeat_timeout,
1325            "start_to_close_timeout",
1326            self.start_to_close_timeout,
1327        )?;
1328        validate_timeout_order(
1329            "start_to_close_timeout",
1330            self.start_to_close_timeout,
1331            "schedule_to_close_timeout",
1332            self.schedule_to_close_timeout,
1333        )?;
1334        validate_timeout_order(
1335            "schedule_to_start_timeout",
1336            self.schedule_to_start_timeout,
1337            "schedule_to_close_timeout",
1338            self.schedule_to_close_timeout,
1339        )?;
1340
1341        Ok(ValidatedActivityOptions {
1342            task_queue: self.task_queue.clone(),
1343            retry_policy: self
1344                .retry_policy
1345                .as_ref()
1346                .map(validate_activity_retry_policy)
1347                .transpose()?,
1348            start_to_close_timeout: timeout_seconds(
1349                "start_to_close_timeout",
1350                self.start_to_close_timeout,
1351            )?,
1352            schedule_to_start_timeout: timeout_seconds(
1353                "schedule_to_start_timeout",
1354                self.schedule_to_start_timeout,
1355            )?,
1356            schedule_to_close_timeout: timeout_seconds(
1357                "schedule_to_close_timeout",
1358                self.schedule_to_close_timeout,
1359            )?,
1360            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1361        })
1362    }
1363}
1364
1365/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1366///
1367/// Constructors capture arguments but perform no I/O. The join validates the
1368/// complete tree, attaches the existing parallel-group metadata to every
1369/// ordinary command, schedules all leaves, and then suspends.
1370pub enum ParallelOperation {
1371    Activity {
1372        activity_type: String,
1373        options: ActivityOptions,
1374        arguments: Result<AvroValue>,
1375    },
1376    ChildWorkflow {
1377        workflow_type: String,
1378        options: ChildWorkflowOptions,
1379        arguments: Result<AvroValue>,
1380    },
1381    Timer(Duration),
1382    Group(Vec<ParallelOperation>),
1383}
1384
1385impl ParallelOperation {
1386    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1387        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1388    }
1389
1390    pub fn activity_with_options<T: Serialize>(
1391        activity_type: impl Into<String>,
1392        options: ActivityOptions,
1393        args: T,
1394    ) -> Self {
1395        Self::Activity {
1396            activity_type: activity_type.into(),
1397            options,
1398            arguments: AvroValue::from_serialize(&args),
1399        }
1400    }
1401
1402    pub fn child_workflow<T: Serialize>(
1403        workflow_type: impl Into<String>,
1404        options: ChildWorkflowOptions,
1405        args: T,
1406    ) -> Self {
1407        Self::ChildWorkflow {
1408            workflow_type: workflow_type.into(),
1409            options,
1410            arguments: AvroValue::from_serialize(&args),
1411        }
1412    }
1413
1414    pub fn timer(duration: Duration) -> Self {
1415        Self::Timer(duration)
1416    }
1417
1418    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1419        Self::Group(operations)
1420    }
1421}
1422
1423#[derive(Clone, Debug)]
1424struct ValidatedActivityOptions {
1425    task_queue: Option<String>,
1426    retry_policy: Option<Value>,
1427    start_to_close_timeout: Option<u64>,
1428    schedule_to_start_timeout: Option<u64>,
1429    schedule_to_close_timeout: Option<u64>,
1430    heartbeat_timeout: Option<u64>,
1431}
1432
1433fn validate_timeout_order(
1434    smaller_name: &'static str,
1435    smaller: Option<Duration>,
1436    larger_name: &'static str,
1437    larger: Option<Duration>,
1438) -> std::result::Result<(), ActivityOptionsError> {
1439    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1440        return Err(ActivityOptionsError::new(
1441            ActivityOptionsErrorKind::TimeoutOrder,
1442            Some(smaller_name),
1443            format!("{smaller_name} must be <= {larger_name}"),
1444        ));
1445    }
1446    Ok(())
1447}
1448
1449fn timeout_seconds(
1450    field: &'static str,
1451    value: Option<Duration>,
1452) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1453    value
1454        .map(|value| {
1455            activity_protocol_seconds(value).ok_or_else(|| {
1456                ActivityOptionsError::new(
1457                    ActivityOptionsErrorKind::TimeoutOverflow,
1458                    Some(field),
1459                    format!("{field} is too large for the worker protocol"),
1460                )
1461            })
1462        })
1463        .transpose()
1464}
1465
1466fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1467    value
1468        .as_secs()
1469        .checked_add(u64::from(value.subsec_nanos() > 0))
1470}
1471
1472fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1473    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1474}
1475
1476fn validate_activity_retry_policy(
1477    policy: &ActivityRetryPolicy,
1478) -> std::result::Result<Value, ActivityOptionsError> {
1479    if policy.max_attempts.is_none()
1480        && policy.backoff.is_none()
1481        && policy.non_retryable_error_types.is_empty()
1482    {
1483        return Err(ActivityOptionsError::new(
1484            ActivityOptionsErrorKind::EmptyRetryPolicy,
1485            Some("retry_policy"),
1486            "retry_policy must configure at least one field",
1487        ));
1488    }
1489    if policy.max_attempts == Some(0) {
1490        return Err(ActivityOptionsError::new(
1491            ActivityOptionsErrorKind::InvalidMaxAttempts,
1492            Some("retry_policy.max_attempts"),
1493            "max_attempts must be >= 1",
1494        ));
1495    }
1496    if policy
1497        .non_retryable_error_types
1498        .iter()
1499        .any(|error_type| error_type.trim().is_empty())
1500    {
1501        return Err(ActivityOptionsError::new(
1502            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1503            Some("retry_policy.non_retryable_error_types"),
1504            "non_retryable_error_types must not contain empty values",
1505        ));
1506    }
1507
1508    let backoff_seconds = match &policy.backoff {
1509        None => None,
1510        Some(backoff) => {
1511            let max_attempts = policy.max_attempts.ok_or_else(|| {
1512                ActivityOptionsError::new(
1513                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1514                    Some("retry_policy.backoff"),
1515                    "backoff requires max_attempts",
1516                )
1517            })?;
1518            let retry_count = max_attempts.saturating_sub(1) as usize;
1519            let intervals = match backoff {
1520                ActivityBackoff::Explicit(intervals) => {
1521                    if intervals.len() > retry_count {
1522                        return Err(ActivityOptionsError::new(
1523                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1524                            Some("retry_policy.backoff"),
1525                            "backoff interval count must not exceed max_attempts - 1",
1526                        ));
1527                    }
1528                    intervals.clone()
1529                }
1530                ActivityBackoff::Exponential {
1531                    initial_interval,
1532                    coefficient,
1533                    maximum_interval,
1534                } => {
1535                    if *coefficient < 1 {
1536                        return Err(ActivityOptionsError::new(
1537                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1538                            Some("retry_policy.backoff.coefficient"),
1539                            "backoff coefficient must be >= 1",
1540                        ));
1541                    }
1542                    if retry_count > 10_000 {
1543                        return Err(ActivityOptionsError::new(
1544                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1545                            Some("retry_policy.max_attempts"),
1546                            "generated backoff supports at most 10000 retry intervals",
1547                        ));
1548                    }
1549                    let mut current = *initial_interval;
1550                    let mut intervals = Vec::with_capacity(retry_count);
1551                    for _ in 0..retry_count {
1552                        let interval = maximum_interval
1553                            .map(|maximum| current.min(maximum))
1554                            .unwrap_or(current);
1555                        intervals.push(interval);
1556                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1557                            break;
1558                        }
1559                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1560                            ActivityOptionsError::new(
1561                                ActivityOptionsErrorKind::BackoffOverflow,
1562                                Some("retry_policy.backoff"),
1563                                "generated backoff interval overflowed",
1564                            )
1565                        })?;
1566                    }
1567                    intervals
1568                }
1569            };
1570            Some(
1571                intervals
1572                    .into_iter()
1573                    .map(|interval| {
1574                        activity_protocol_seconds(interval).ok_or_else(|| {
1575                            ActivityOptionsError::new(
1576                                ActivityOptionsErrorKind::BackoffOverflow,
1577                                Some("retry_policy.backoff"),
1578                                "backoff interval is too large for the worker protocol",
1579                            )
1580                        })
1581                    })
1582                    .collect::<std::result::Result<Vec<_>, _>>()?,
1583            )
1584        }
1585    };
1586
1587    let mut encoded = serde_json::Map::new();
1588    if let Some(max_attempts) = policy.max_attempts {
1589        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1590    }
1591    if let Some(backoff_seconds) = backoff_seconds {
1592        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1593    }
1594    if !policy.non_retryable_error_types.is_empty() {
1595        let mut canonical_error_types = Vec::new();
1596        for error_type in policy
1597            .non_retryable_error_types
1598            .iter()
1599            .map(|error_type| error_type.trim())
1600        {
1601            if !canonical_error_types.contains(&error_type) {
1602                canonical_error_types.push(error_type);
1603            }
1604        }
1605        encoded.insert(
1606            "non_retryable_error_types".to_string(),
1607            json!(canonical_error_types),
1608        );
1609    }
1610    Ok(Value::Object(encoded))
1611}
1612
1613/// A stable, machine-readable failure raised when workflow code no longer
1614/// reconstructs the durable command stream recorded in history.
1615#[derive(Clone, Debug, Error)]
1616#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1617pub struct ReplayFailure {
1618    pub reason: String,
1619    pub sequence: Option<u64>,
1620    pub expected: Option<String>,
1621    pub actual: Option<String>,
1622    pub message: String,
1623}
1624
1625impl ReplayFailure {
1626    fn new(
1627        reason: impl Into<String>,
1628        sequence: Option<u64>,
1629        expected: Option<String>,
1630        actual: Option<String>,
1631        message: impl Into<String>,
1632    ) -> Self {
1633        Self {
1634            reason: reason.into(),
1635            sequence,
1636            expected,
1637            actual,
1638            message: message.into(),
1639        }
1640    }
1641}
1642
1643/// A stable, machine-readable workflow query or query-task settlement failure.
1644#[derive(Clone, Debug, Error)]
1645#[error("query failed ({reason}, HTTP {status}): {message}")]
1646pub struct QueryFailure {
1647    pub status: u16,
1648    pub reason: String,
1649    pub message: String,
1650    pub body: Value,
1651}
1652
1653/// A stable failure returned when a server rejects an SDK protocol version.
1654#[derive(Clone, Debug, Error)]
1655#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1656pub struct ProtocolFailure {
1657    pub status: u16,
1658    pub reason: String,
1659    pub message: String,
1660    pub supported_version: Option<String>,
1661    pub requested_version: Option<String>,
1662    pub body: Value,
1663}
1664
1665#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1666pub struct PayloadEnvelope {
1667    pub codec: String,
1668    pub blob: String,
1669}
1670
1671impl PayloadEnvelope {
1672    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1673        encode_payload(value, DEFAULT_CODEC)
1674    }
1675
1676    /// Encode an explicit typed value, including the bytes branch that JSON
1677    /// serialization cannot represent.
1678    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1679        encode_avro_value(value)
1680    }
1681}
1682
1683/// Native adapter for the fixed language-neutral Avro Value schema.
1684#[derive(Clone, Debug)]
1685pub enum AvroValue {
1686    Null,
1687    Boolean(bool),
1688    Long(i64),
1689    Double(f64),
1690    Bytes(Vec<u8>),
1691    String(String),
1692    Array(Vec<AvroValue>),
1693    Map(BTreeMap<String, AvroValue>),
1694}
1695
1696impl PartialEq for AvroValue {
1697    fn eq(&self, other: &Self) -> bool {
1698        match (self, other) {
1699            (Self::Null, Self::Null) => true,
1700            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1701            (Self::Long(left), Self::Long(right)) => left == right,
1702            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1703            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1704            (Self::String(left), Self::String(right)) => left == right,
1705            (Self::Array(left), Self::Array(right)) => left == right,
1706            (Self::Map(left), Self::Map(right)) => left == right,
1707            _ => false,
1708        }
1709    }
1710}
1711
1712impl AvroValue {
1713    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1714        Self::from_serde_value(
1715            serde_value::to_value(value).map_err(|error| {
1716                Error::Codec(format!("could not adapt value for Avro: {error}"))
1717            })?,
1718        )
1719    }
1720
1721    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1722        use serde_value::Value as SerdeValue;
1723
1724        match value {
1725            SerdeValue::Unit => Ok(Self::Null),
1726            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1727            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1728            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1729            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1730            SerdeValue::I64(value) => Ok(Self::Long(value)),
1731            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1732            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1733            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1734            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1735                Error::Codec(
1736                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1737                        .to_string(),
1738                )
1739            }),
1740            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1741            SerdeValue::F64(value) => Self::finite_double(value),
1742            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1743            SerdeValue::String(value) => Ok(Self::String(value)),
1744            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1745            SerdeValue::Option(None) => Ok(Self::Null),
1746            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1747                Self::from_serde_value(*value)
1748            }
1749            SerdeValue::Seq(values) => values
1750                .into_iter()
1751                .map(Self::from_serde_value)
1752                .collect::<Result<Vec<_>>>()
1753                .map(Self::Array),
1754            SerdeValue::Map(values) => values
1755                .into_iter()
1756                .map(|(key, value)| {
1757                    let SerdeValue::String(key) = key else {
1758                        return Err(Error::Codec(
1759                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1760                        ));
1761                    };
1762
1763                    Ok((key, Self::from_serde_value(value)?))
1764                })
1765                .collect::<Result<BTreeMap<_, _>>>()
1766                .map(Self::Map),
1767        }
1768    }
1769
1770    fn finite_double(value: f64) -> Result<Self> {
1771        if !value.is_finite() {
1772            return Err(Error::Codec(
1773                "non_finite_float: Avro Value doubles must be finite".to_string(),
1774            ));
1775        }
1776
1777        Ok(Self::Double(value))
1778    }
1779
1780    fn into_json(self) -> Result<Value> {
1781        match self {
1782            Self::Null => Ok(Value::Null),
1783            Self::Boolean(value) => Ok(Value::Bool(value)),
1784            Self::Long(value) => Ok(Value::Number(value.into())),
1785            Self::Double(value) => serde_json::Number::from_f64(value)
1786                .map(Value::Number)
1787                .ok_or_else(|| {
1788                    Error::Codec(
1789                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1790                    )
1791                }),
1792            Self::Bytes(value) => Ok(json!({
1793                "$type": "bytes",
1794                "base64": BASE64.encode(value),
1795            })),
1796            Self::String(value) => Ok(Value::String(value)),
1797            Self::Array(values) => values
1798                .into_iter()
1799                .map(Self::into_json)
1800                .collect::<Result<Vec<_>>>()
1801                .map(Value::Array),
1802            Self::Map(values) => values
1803                .into_iter()
1804                .map(|(key, value)| Ok((key, value.into_json()?)))
1805                .collect::<Result<serde_json::Map<_, _>>>()
1806                .map(Value::Object),
1807        }
1808    }
1809
1810    fn into_serde_value(self) -> serde_value::Value {
1811        use serde_value::Value as SerdeValue;
1812
1813        match self {
1814            Self::Null => SerdeValue::Unit,
1815            Self::Boolean(value) => SerdeValue::Bool(value),
1816            Self::Long(value) => SerdeValue::I64(value),
1817            Self::Double(value) => SerdeValue::F64(value),
1818            Self::Bytes(value) => SerdeValue::Bytes(value),
1819            Self::String(value) => SerdeValue::String(value),
1820            Self::Array(values) => {
1821                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1822            }
1823            Self::Map(values) => SerdeValue::Map(
1824                values
1825                    .into_iter()
1826                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1827                    .collect(),
1828            ),
1829        }
1830    }
1831
1832    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1833        self.into_serde_value().deserialize_into().map_err(|error| {
1834            Error::Codec(format!(
1835                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1836            ))
1837        })
1838    }
1839}
1840
1841impl Serialize for AvroValue {
1842    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1843    where
1844        S: Serializer,
1845    {
1846        match self {
1847            Self::Null => serializer.serialize_unit(),
1848            Self::Boolean(value) => serializer.serialize_bool(*value),
1849            Self::Long(value) => serializer.serialize_i64(*value),
1850            Self::Double(value) => serializer.serialize_f64(*value),
1851            Self::Bytes(value) => serializer.serialize_bytes(value),
1852            Self::String(value) => serializer.serialize_str(value),
1853            Self::Array(values) => {
1854                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
1855                for value in values {
1856                    sequence.serialize_element(value)?;
1857                }
1858                sequence.end()
1859            }
1860            Self::Map(values) => {
1861                let mut map = serializer.serialize_map(Some(values.len()))?;
1862                for (key, value) in values {
1863                    map.serialize_entry(key, value)?;
1864                }
1865                map.end()
1866            }
1867        }
1868    }
1869}
1870
1871pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
1872    let mut bytes = Vec::new();
1873    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
1874    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
1875    encode_avro_value_datum(&mut bytes, value)?;
1876    Ok(PayloadEnvelope {
1877        codec: DEFAULT_CODEC.to_string(),
1878        blob: BASE64.encode(bytes),
1879    })
1880}
1881
1882pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
1883    if envelope.codec != DEFAULT_CODEC {
1884        return Err(unsupported_payload_codec(&envelope.codec));
1885    }
1886    decode_avro_value_blob(&envelope.blob)
1887}
1888
1889pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
1890    let blob = match codec {
1891        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
1892        other => return Err(unsupported_payload_codec(other)),
1893    };
1894
1895    Ok(PayloadEnvelope {
1896        codec: codec.to_string(),
1897        blob,
1898    })
1899}
1900
1901pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
1902    match envelope.codec.as_str() {
1903        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
1904        other => Err(unsupported_payload_codec(other)),
1905    }
1906}
1907
1908fn handler_type_error<T>(
1909    handler_kind: HandlerKind,
1910    handler_name: &str,
1911    value_kind: HandlerValueKind,
1912    message: impl Into<String>,
1913) -> Error {
1914    Error::HandlerType {
1915        handler_kind,
1916        handler_name: handler_name.to_string(),
1917        value_kind,
1918        rust_type: type_name::<T>(),
1919        message: message.into(),
1920    }
1921}
1922
1923fn decode_handler_input<T: DeserializeOwned>(
1924    arguments: AvroValue,
1925    handler_kind: HandlerKind,
1926    handler_name: &str,
1927) -> Result<T> {
1928    let argument = match arguments {
1929        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
1930            arguments.pop().expect("one typed handler argument")
1931        }
1932        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
1933        AvroValue::Array(arguments) => {
1934            return Err(handler_type_error::<T>(
1935                handler_kind,
1936                handler_name,
1937                HandlerValueKind::Input,
1938                format!(
1939                    "typed handlers accept one request value, but the task carried {} arguments",
1940                    arguments.len()
1941                ),
1942            ));
1943        }
1944        argument => argument,
1945    };
1946
1947    argument.deserialize().map_err(|error| {
1948        handler_type_error::<T>(
1949            handler_kind,
1950            handler_name,
1951            HandlerValueKind::Input,
1952            error.to_string(),
1953        )
1954    })
1955}
1956
1957fn encode_handler_result<T: Serialize>(
1958    result: &T,
1959    handler_kind: HandlerKind,
1960    handler_name: &str,
1961) -> Result<AvroValue> {
1962    AvroValue::from_serialize(result).map_err(|error| {
1963        handler_type_error::<T>(
1964            handler_kind,
1965            handler_name,
1966            HandlerValueKind::Result,
1967            error.to_string(),
1968        )
1969    })
1970}
1971
1972fn decode_handler_result<T: DeserializeOwned>(
1973    result: AvroValue,
1974    handler_kind: HandlerKind,
1975    handler_name: &str,
1976) -> Result<T> {
1977    result.deserialize().map_err(|error| {
1978        handler_type_error::<T>(
1979            handler_kind,
1980            handler_name,
1981            HandlerValueKind::Result,
1982            error.to_string(),
1983        )
1984    })
1985}
1986
1987#[cfg(test)]
1988fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
1989    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
1990}
1991
1992fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
1993    validate_payload_codec(fallback_codec)?;
1994
1995    if value.is_null() {
1996        return Ok(Value::Null);
1997    }
1998
1999    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2000        return decode_blob(blob, codec);
2001    }
2002
2003    if let Some(blob) = value.as_str() {
2004        return decode_blob(blob, fallback_codec);
2005    }
2006
2007    Err(untagged_payload_value())
2008}
2009
2010fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2011    let envelope = match codec {
2012        DEFAULT_CODEC => encode_avro_value(value)?,
2013        other => return Err(unsupported_payload_codec(other)),
2014    };
2015    Ok(serde_json::to_value(envelope)?)
2016}
2017
2018fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2019    validate_payload_codec(fallback_codec)?;
2020
2021    if value.is_null() {
2022        return Ok(AvroValue::Null);
2023    }
2024
2025    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2026        validate_payload_codec(codec)?;
2027        return decode_avro_value_blob(blob);
2028    }
2029
2030    if let Some(blob) = value.as_str() {
2031        return match fallback_codec {
2032            DEFAULT_CODEC => decode_avro_value_blob(blob),
2033            other => Err(unsupported_payload_codec(other)),
2034        };
2035    }
2036
2037    Err(untagged_payload_value())
2038}
2039
2040fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2041    match value {
2042        AvroValue::Null => AvroValue::Array(Vec::new()),
2043        AvroValue::Array(_) => value,
2044        other => AvroValue::Array(vec![other]),
2045    }
2046}
2047
2048fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2049    match codec {
2050        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2051        other => Err(unsupported_payload_codec(other)),
2052    }
2053}
2054
2055fn validate_payload_codec(codec: &str) -> Result<()> {
2056    match codec {
2057        DEFAULT_CODEC => Ok(()),
2058        MISSING_TASK_PAYLOAD_CODEC => {
2059            Err(invalid_task_payload_codec("task payload_codec is missing"))
2060        }
2061        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2062        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2063            "task payload_codec must be a string",
2064        )),
2065        other => Err(unsupported_payload_codec(other)),
2066    }
2067}
2068
2069fn invalid_task_payload_codec(reason: &str) -> Error {
2070    Error::Codec(format!(
2071        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2072    ))
2073}
2074
2075fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2076    let Some(object) = value.as_object() else {
2077        return Ok(None);
2078    };
2079    if !object.contains_key("codec") && !object.contains_key("blob") {
2080        return Ok(None);
2081    }
2082
2083    let codec = object
2084        .get("codec")
2085        .and_then(Value::as_str)
2086        .ok_or_else(invalid_payload_envelope)?;
2087    validate_payload_codec(codec)?;
2088    let blob = object
2089        .get("blob")
2090        .and_then(Value::as_str)
2091        .ok_or_else(invalid_payload_envelope)?;
2092    Ok(Some((codec, blob)))
2093}
2094
2095fn invalid_payload_envelope() -> Error {
2096    Error::Codec(
2097        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2098            .to_string(),
2099    )
2100}
2101
2102fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2103    for command in commands {
2104        let Some(command) = command.as_object() else {
2105            continue;
2106        };
2107        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2108            continue;
2109        };
2110        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2111            continue;
2112        };
2113
2114        if let Some(codec) = command.get("payload_codec") {
2115            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2116            validate_payload_codec(codec)?;
2117        }
2118
2119        let payload = command
2120            .get(payload_field)
2121            .ok_or_else(invalid_payload_envelope)?;
2122        validate_outbound_payload_envelope(payload)?;
2123    }
2124    Ok(())
2125}
2126
2127fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2128    if commands
2129        .iter()
2130        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2131    {
2132        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2133    } else if commands.iter().any(|command| {
2134        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2135    }) {
2136        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2137    } else {
2138        WORKER_PROTOCOL_VERSION
2139    }
2140}
2141
2142fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2143    match command_type {
2144        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2145        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2146        "start_service_operation" => Some("request_payload"),
2147        "upsert_memo" => Some("entries"),
2148        _ => None,
2149    }
2150}
2151
2152fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2153    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2154        return Err(untagged_payload_value());
2155    };
2156    validate_payload_codec(codec)?;
2157    decode_avro_value_blob(blob)?;
2158    Ok(())
2159}
2160
2161fn unsupported_payload_codec(codec: &str) -> Error {
2162    Error::Codec(format!(
2163        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2164    ))
2165}
2166
2167fn untagged_payload_value() -> Error {
2168    Error::Codec(
2169        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2170            .to_string(),
2171    )
2172}
2173
2174fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2175    let bytes = BASE64.decode(blob).map_err(|err| {
2176        Error::Codec(format!(
2177            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2178        ))
2179    })?;
2180
2181    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2182        return Err(unsupported_payload_codec("json"));
2183    }
2184
2185    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2186        return Err(Error::Codec(
2187            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2188        ));
2189    }
2190
2191    let fingerprint: [u8; 8] = bytes[2..10]
2192        .try_into()
2193        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2194    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2195        return Err(Error::Codec(format!(
2196            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2197            fingerprint
2198                .iter()
2199                .map(|byte| format!("{byte:02x}"))
2200                .collect::<String>()
2201        )));
2202    }
2203
2204    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2205    // The current fingerprint selects the current immutable schema, so reader
2206    // resolution would only re-walk the same recursive union. Future retained
2207    // writer fingerprints supply a distinct reader schema in this branch.
2208    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2209    if datum_reader.truncated {
2210        return Err(Error::Codec(
2211            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2212        ));
2213    }
2214    let datum = datum.map_err(|err| {
2215        Error::Codec(format!(
2216            "invalid_payload_framing: malformed Avro Value datum: {err}"
2217        ))
2218    })?;
2219    if datum_reader.remaining() != 0 {
2220        return Err(Error::Codec(format!(
2221            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2222            datum_reader.remaining()
2223        )));
2224    }
2225    avro_value_from_datum(datum)
2226}
2227
2228struct StrictAvroDatumReader<'a> {
2229    bytes: &'a [u8],
2230    offset: usize,
2231    truncated: bool,
2232}
2233
2234impl<'a> StrictAvroDatumReader<'a> {
2235    fn new(bytes: &'a [u8]) -> Self {
2236        Self {
2237            bytes,
2238            offset: 0,
2239            truncated: false,
2240        }
2241    }
2242
2243    fn remaining(&self) -> usize {
2244        self.bytes.len() - self.offset
2245    }
2246}
2247
2248impl Read for StrictAvroDatumReader<'_> {
2249    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2250        let count = buffer.len().min(self.remaining());
2251        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2252        self.offset += count;
2253        if count < buffer.len() {
2254            self.truncated = true;
2255        }
2256
2257        Ok(count)
2258    }
2259}
2260
2261fn encode_avro_long(bytes: &mut Vec<u8>, value: i64) {
2262    let mut value = ((value as u64) << 1) ^ ((value >> 63) as u64);
2263    loop {
2264        if value & !0x7f == 0 {
2265            bytes.push(value as u8);
2266            break;
2267        }
2268        bytes.push(((value & 0x7f) | 0x80) as u8);
2269        value >>= 7;
2270    }
2271}
2272
2273fn encode_avro_size(bytes: &mut Vec<u8>, size: usize) -> Result<()> {
2274    let size = i64::try_from(size)
2275        .map_err(|_| Error::Codec("avro_value_encode_failed: collection too large".to_string()))?;
2276    encode_avro_long(bytes, size);
2277    Ok(())
2278}
2279
2280fn encode_avro_bytes(bytes: &mut Vec<u8>, value: &[u8]) -> Result<()> {
2281    encode_avro_size(bytes, value.len())?;
2282    bytes.extend_from_slice(value);
2283    Ok(())
2284}
2285
2286fn encode_avro_string(bytes: &mut Vec<u8>, value: &str) -> Result<()> {
2287    encode_avro_bytes(bytes, value.as_bytes())
2288}
2289
2290fn encode_avro_value_datum(bytes: &mut Vec<u8>, value: &AvroValue) -> Result<()> {
2291    match value {
2292        AvroValue::Null => encode_avro_long(bytes, 0),
2293        AvroValue::Boolean(value) => {
2294            encode_avro_long(bytes, 1);
2295            bytes.push(u8::from(*value));
2296        }
2297        AvroValue::Long(value) => {
2298            encode_avro_long(bytes, 2);
2299            encode_avro_long(bytes, *value);
2300        }
2301        AvroValue::Double(value) => {
2302            if !value.is_finite() {
2303                return Err(Error::Codec(
2304                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2305                ));
2306            }
2307            encode_avro_long(bytes, 3);
2308            bytes.extend_from_slice(&value.to_le_bytes());
2309        }
2310        AvroValue::Bytes(value) => {
2311            encode_avro_long(bytes, 4);
2312            encode_avro_bytes(bytes, value)?;
2313        }
2314        AvroValue::String(value) => {
2315            encode_avro_long(bytes, 5);
2316            encode_avro_string(bytes, value)?;
2317        }
2318        AvroValue::Array(values) => {
2319            encode_avro_long(bytes, 6);
2320            if !values.is_empty() {
2321                encode_avro_size(bytes, values.len())?;
2322                for value in values {
2323                    encode_avro_value_datum(bytes, value)?;
2324                }
2325            }
2326            encode_avro_long(bytes, 0);
2327        }
2328        AvroValue::Map(values) => {
2329            encode_avro_long(bytes, 7);
2330            if !values.is_empty() {
2331                encode_avro_size(bytes, values.len())?;
2332                for (key, value) in values {
2333                    encode_avro_string(bytes, key)?;
2334                    encode_avro_value_datum(bytes, value)?;
2335                }
2336            }
2337            encode_avro_long(bytes, 0);
2338        }
2339    }
2340    Ok(())
2341}
2342
2343fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2344    let AvroDatum::Record(mut outer) = datum else {
2345        return Err(Error::Codec(
2346            "invalid_payload_framing: datum is not a Value record".to_string(),
2347        ));
2348    };
2349    let (_, branch) = outer
2350        .pop()
2351        .filter(|(name, _)| name == "value")
2352        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2353    let AvroDatum::Union(_, branch) = branch else {
2354        return Err(Error::Codec(
2355            "invalid_payload_framing: invalid Value union".to_string(),
2356        ));
2357    };
2358    match *branch {
2359        AvroDatum::Null => Ok(AvroValue::Null),
2360        AvroDatum::Record(mut fields) => {
2361            let (name, value) = fields.pop().ok_or_else(|| {
2362                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2363            })?;
2364            match (name.as_str(), value) {
2365                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2366                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2367                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2368                    Ok(AvroValue::Double(value))
2369                }
2370                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2371                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2372                ("items", AvroDatum::Array(values)) => values
2373                    .into_iter()
2374                    .map(avro_value_from_datum)
2375                    .collect::<Result<Vec<_>>>()
2376                    .map(AvroValue::Array),
2377                ("entries", AvroDatum::Map(values)) => values
2378                    .into_iter()
2379                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2380                    .collect::<Result<BTreeMap<_, _>>>()
2381                    .map(AvroValue::Map),
2382                _ => Err(Error::Codec(
2383                    "invalid_payload_framing: unknown Value branch".to_string(),
2384                )),
2385            }
2386        }
2387        _ => Err(Error::Codec(
2388            "invalid_payload_framing: invalid Value branch".to_string(),
2389        )),
2390    }
2391}
2392
2393fn avro_value_schema() -> Result<&'static Schema> {
2394    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2395        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2396            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2397    }) {
2398        Ok(schema) => Ok(schema),
2399        Err(message) => Err(Error::Codec(message.clone())),
2400    }
2401}
2402
2403#[derive(Clone, Debug)]
2404pub struct Client {
2405    http: reqwest::Client,
2406    base_url: String,
2407    token: Option<String>,
2408    control_token: Option<String>,
2409    worker_token: Option<String>,
2410    namespace: String,
2411}
2412
2413impl Client {
2414    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2415        Self::builder(base_url).build()
2416    }
2417
2418    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2419        ClientBuilder {
2420            base_url: base_url.into(),
2421            token: None,
2422            control_token: None,
2423            worker_token: None,
2424            namespace: "default".to_string(),
2425            timeout: Duration::from_secs(60),
2426        }
2427    }
2428
2429    pub async fn health(&self) -> Result<Value> {
2430        self.request_json(
2431            reqwest::Method::GET,
2432            "/health",
2433            RequestProtocol::ControlPlane,
2434            Option::<&Value>::None,
2435        )
2436        .await
2437    }
2438
2439    pub async fn cluster_info(&self) -> Result<Value> {
2440        self.request_json(
2441            reqwest::Method::GET,
2442            "/cluster/info",
2443            RequestProtocol::ControlPlane,
2444            Option::<&Value>::None,
2445        )
2446        .await
2447    }
2448
2449    pub async fn start_workflow<T: Serialize>(
2450        &self,
2451        workflow_type: &str,
2452        task_queue: &str,
2453        workflow_id: &str,
2454        input: T,
2455    ) -> Result<WorkflowHandle> {
2456        self.start_workflow_with_options(
2457            workflow_type,
2458            task_queue,
2459            workflow_id,
2460            WorkflowStartOptions::default(),
2461            input,
2462        )
2463        .await
2464    }
2465
2466    /// Start a workflow with explicit server-enforced execution and run
2467    /// deadlines.
2468    pub async fn start_workflow_with_options<T: Serialize>(
2469        &self,
2470        workflow_type: &str,
2471        task_queue: &str,
2472        workflow_id: &str,
2473        options: WorkflowStartOptions,
2474        input: T,
2475    ) -> Result<WorkflowHandle> {
2476        options.validate()?;
2477        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2478        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2479        let body = json!({
2480            "workflow_id": workflow_id,
2481            "workflow_type": workflow_type,
2482            "task_queue": task_queue,
2483            "input": input_envelope,
2484            "execution_timeout_seconds": options.execution_timeout_seconds,
2485            "run_timeout_seconds": options.run_timeout_seconds
2486        });
2487
2488        let data: Value = self
2489            .request_json(
2490                reqwest::Method::POST,
2491                "/workflows",
2492                RequestProtocol::ControlPlane,
2493                Some(&body),
2494            )
2495            .await?;
2496
2497        Ok(WorkflowHandle {
2498            client: self.clone(),
2499            workflow_id: data
2500                .get("workflow_id")
2501                .and_then(Value::as_str)
2502                .unwrap_or(workflow_id)
2503                .to_string(),
2504            run_id: data
2505                .get("run_id")
2506                .and_then(Value::as_str)
2507                .map(str::to_string),
2508            workflow_type: data
2509                .get("workflow_type")
2510                .and_then(Value::as_str)
2511                .unwrap_or(workflow_type)
2512                .to_string(),
2513        })
2514    }
2515
2516    pub async fn signal_workflow<T: Serialize>(
2517        &self,
2518        workflow_id: &str,
2519        signal_name: &str,
2520        input: T,
2521    ) -> Result<Value> {
2522        self.signal_workflow_target(workflow_id, None, signal_name, input)
2523            .await
2524    }
2525
2526    /// Signal only if `run_id` is still the current run for this instance.
2527    pub async fn signal_workflow_run<T: Serialize>(
2528        &self,
2529        workflow_id: &str,
2530        run_id: &str,
2531        signal_name: &str,
2532        input: T,
2533    ) -> Result<Value> {
2534        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2535            .await
2536    }
2537
2538    async fn signal_workflow_target<T: Serialize>(
2539        &self,
2540        workflow_id: &str,
2541        run_id: Option<&str>,
2542        signal_name: &str,
2543        input: T,
2544    ) -> Result<Value> {
2545        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2546        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2547        let body = json!({
2548            "input": input_envelope
2549        });
2550        let path = match run_id {
2551            Some(run_id) => {
2552                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2553            }
2554            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2555        };
2556        self.request_json(
2557            reqwest::Method::POST,
2558            &path,
2559            RequestProtocol::ControlPlane,
2560            Some(&body),
2561        )
2562        .await
2563    }
2564
2565    /// Request cooperative cancellation of the current run for an instance.
2566    pub async fn cancel_workflow(
2567        &self,
2568        workflow_id: &str,
2569        options: WorkflowCommandOptions,
2570    ) -> Result<WorkflowCommandResult> {
2571        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2572            .await
2573    }
2574
2575    /// Request cooperative cancellation only if `run_id` is still current.
2576    pub async fn cancel_workflow_run(
2577        &self,
2578        workflow_id: &str,
2579        run_id: &str,
2580        options: WorkflowCommandOptions,
2581    ) -> Result<WorkflowCommandResult> {
2582        self.workflow_command(
2583            workflow_id,
2584            Some(run_id),
2585            WorkflowCommandKind::Cancel,
2586            options,
2587        )
2588        .await
2589    }
2590
2591    /// Forcefully terminate the current run for an instance.
2592    pub async fn terminate_workflow(
2593        &self,
2594        workflow_id: &str,
2595        options: WorkflowCommandOptions,
2596    ) -> Result<WorkflowCommandResult> {
2597        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2598            .await
2599    }
2600
2601    /// Forcefully terminate only if `run_id` is still current.
2602    pub async fn terminate_workflow_run(
2603        &self,
2604        workflow_id: &str,
2605        run_id: &str,
2606        options: WorkflowCommandOptions,
2607    ) -> Result<WorkflowCommandResult> {
2608        self.workflow_command(
2609            workflow_id,
2610            Some(run_id),
2611            WorkflowCommandKind::Terminate,
2612            options,
2613        )
2614        .await
2615    }
2616
2617    async fn workflow_command(
2618        &self,
2619        workflow_id: &str,
2620        run_id: Option<&str>,
2621        command: WorkflowCommandKind,
2622        options: WorkflowCommandOptions,
2623    ) -> Result<WorkflowCommandResult> {
2624        let path = match run_id {
2625            Some(run_id) => format!(
2626                "/workflows/{workflow_id}/runs/{run_id}/{}",
2627                command.as_str()
2628            ),
2629            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2630        };
2631        let data = match self
2632            .request_json(
2633                reqwest::Method::POST,
2634                &path,
2635                RequestProtocol::ControlPlane,
2636                Some(&options),
2637            )
2638            .await
2639        {
2640            Ok(data) => data,
2641            Err(Error::Http { status, body }) => {
2642                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2643                    command,
2644                    status,
2645                    body,
2646                    workflow_id,
2647                    run_id,
2648                )));
2649            }
2650            Err(error) => return Err(error),
2651        };
2652
2653        Ok(workflow_command_result(command, data, workflow_id, run_id))
2654    }
2655
2656    /// Execute a named, read-only query against a running or completed workflow.
2657    ///
2658    /// Arguments and results use the platform payload envelope. Server and
2659    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2660    /// reason, HTTP status, and original response body.
2661    pub async fn query_workflow<T: Serialize>(
2662        &self,
2663        workflow_id: &str,
2664        query_name: &str,
2665        input: T,
2666    ) -> Result<Value> {
2667        self.query_workflow_target(workflow_id, None, query_name, input)
2668            .await
2669    }
2670
2671    /// Query only if `run_id` is still current, preventing accidental retargeting.
2672    pub async fn query_workflow_run<T: Serialize>(
2673        &self,
2674        workflow_id: &str,
2675        run_id: &str,
2676        query_name: &str,
2677        input: T,
2678    ) -> Result<Value> {
2679        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2680            .await
2681    }
2682
2683    /// Query a workflow and return the lossless fixed Avro Value result.
2684    pub async fn query_workflow_avro_value<T: Serialize>(
2685        &self,
2686        workflow_id: &str,
2687        query_name: &str,
2688        input: T,
2689    ) -> Result<AvroValue> {
2690        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2691            .await
2692    }
2693
2694    /// Query a selected run and return the lossless fixed Avro Value result.
2695    pub async fn query_workflow_run_avro_value<T: Serialize>(
2696        &self,
2697        workflow_id: &str,
2698        run_id: &str,
2699        query_name: &str,
2700        input: T,
2701    ) -> Result<AvroValue> {
2702        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2703            .await
2704    }
2705
2706    async fn query_workflow_avro_value_target<T: Serialize>(
2707        &self,
2708        workflow_id: &str,
2709        run_id: Option<&str>,
2710        query_name: &str,
2711        input: T,
2712    ) -> Result<AvroValue> {
2713        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2714        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2715        let path = match run_id {
2716            Some(run_id) => {
2717                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2718            }
2719            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2720        };
2721        let response: Value = match self
2722            .request_json(
2723                reqwest::Method::POST,
2724                &path,
2725                RequestProtocol::ControlPlane,
2726                Some(&body),
2727            )
2728            .await
2729        {
2730            Ok(response) => response,
2731            Err(Error::Http { status, body }) => {
2732                return Err(Error::QueryFailed(query_failure(status, body)));
2733            }
2734            Err(error) => return Err(error),
2735        };
2736
2737        let envelope = response
2738            .get("result_envelope")
2739            .filter(|envelope| !envelope.is_null())
2740            .ok_or_else(|| {
2741                Error::Codec(
2742                    "missing_payload_envelope: typed query result requires result_envelope"
2743                        .to_string(),
2744                )
2745            })?;
2746        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2747    }
2748
2749    async fn query_workflow_target<T: Serialize>(
2750        &self,
2751        workflow_id: &str,
2752        run_id: Option<&str>,
2753        query_name: &str,
2754        input: T,
2755    ) -> Result<Value> {
2756        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2757        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2758        let body = json!({
2759            "input": input_envelope
2760        });
2761        let path = match run_id {
2762            Some(run_id) => {
2763                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2764            }
2765            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2766        };
2767        let response: Value = match self
2768            .request_json(
2769                reqwest::Method::POST,
2770                &path,
2771                RequestProtocol::ControlPlane,
2772                Some(&body),
2773            )
2774            .await
2775        {
2776            Ok(response) => response,
2777            Err(Error::Http { status, body }) => {
2778                return Err(Error::QueryFailed(query_failure(status, body)));
2779            }
2780            Err(error) => return Err(error),
2781        };
2782
2783        if let Some(envelope) = response
2784            .get("result_envelope")
2785            .filter(|envelope| !envelope.is_null())
2786        {
2787            return decode_wire_value(envelope, DEFAULT_CODEC);
2788        }
2789
2790        Ok(response.get("result").cloned().unwrap_or(Value::Null))
2791    }
2792
2793    /// Send a synchronous update using fixed Avro Value arguments.
2794    pub async fn update_workflow<T: Serialize>(
2795        &self,
2796        workflow_id: &str,
2797        update_name: &str,
2798        input: T,
2799        request_id: Option<&str>,
2800    ) -> Result<Value> {
2801        let response = self
2802            .update_workflow_response(workflow_id, update_name, input, request_id)
2803            .await?;
2804        if let Some(envelope) = response
2805            .get("result_envelope")
2806            .filter(|envelope| !envelope.is_null())
2807        {
2808            return decode_wire_value(envelope, DEFAULT_CODEC);
2809        }
2810        Ok(response.get("result").cloned().unwrap_or(response))
2811    }
2812
2813    /// Send a synchronous update and retain a bytes-capable Avro result.
2814    pub async fn update_workflow_avro_value<T: Serialize>(
2815        &self,
2816        workflow_id: &str,
2817        update_name: &str,
2818        input: T,
2819        request_id: Option<&str>,
2820    ) -> Result<AvroValue> {
2821        let response = self
2822            .update_workflow_response(workflow_id, update_name, input, request_id)
2823            .await?;
2824        let envelope = response
2825            .get("result_envelope")
2826            .filter(|envelope| !envelope.is_null())
2827            .ok_or_else(|| {
2828                Error::Codec(
2829                    "missing_payload_envelope: typed update result requires result_envelope"
2830                        .to_string(),
2831                )
2832            })?;
2833        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2834    }
2835
2836    async fn update_workflow_response<T: Serialize>(
2837        &self,
2838        workflow_id: &str,
2839        update_name: &str,
2840        input: T,
2841        request_id: Option<&str>,
2842    ) -> Result<Value> {
2843        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2844        let mut body = json!({
2845            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
2846            "wait_for": "completed",
2847        });
2848        if let Some(request_id) = request_id {
2849            body["request_id"] = json!(request_id);
2850        }
2851        self.request_json(
2852            reqwest::Method::POST,
2853            &format!("/workflows/{workflow_id}/update/{update_name}"),
2854            RequestProtocol::ControlPlane,
2855            Some(&body),
2856        )
2857        .await
2858    }
2859
2860    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
2861        let path = format!("/workflows/{workflow_id}");
2862        let mut data: WorkflowDescription = self
2863            .request_json(
2864                reqwest::Method::GET,
2865                &path,
2866                RequestProtocol::ControlPlane,
2867                Option::<&Value>::None,
2868            )
2869            .await?;
2870        data.decode_payloads()?;
2871        Ok(data)
2872    }
2873
2874    /// Describe one selected run, including historical terminal runs.
2875    pub async fn describe_workflow_run(
2876        &self,
2877        workflow_id: &str,
2878        run_id: &str,
2879    ) -> Result<WorkflowDescription> {
2880        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
2881        let mut data: WorkflowDescription = self
2882            .request_json(
2883                reqwest::Method::GET,
2884                &path,
2885                RequestProtocol::ControlPlane,
2886                Option::<&Value>::None,
2887            )
2888            .await?;
2889        data.decode_payloads()?;
2890        Ok(data)
2891    }
2892
2893    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
2894        let mut path = format!(
2895            "/workflows/{}/runs/{}/streams",
2896            percent_encode_path_segment(workflow_id),
2897            percent_encode_path_segment(run_id),
2898        );
2899        if let Some(stream_name) = stream_name {
2900            path.push('/');
2901            path.push_str(&percent_encode_path_segment(stream_name));
2902        }
2903        path
2904    }
2905
2906    /// List the run-scoped output streams already opened by a workflow.
2907    pub async fn list_workflow_streams(
2908        &self,
2909        workflow_id: &str,
2910        run_id: &str,
2911    ) -> Result<Vec<WorkflowStreamDescription>> {
2912        let response: WorkflowStreamListResponse = self
2913            .request_json(
2914                reqwest::Method::GET,
2915                &Self::workflow_stream_path(workflow_id, run_id, None),
2916                RequestProtocol::ControlPlane,
2917                Option::<&Value>::None,
2918            )
2919            .await?;
2920        Ok(response.streams)
2921    }
2922
2923    /// Describe stream lifecycle, offsets, pending count, and terminal error.
2924    pub async fn describe_workflow_stream(
2925        &self,
2926        workflow_id: &str,
2927        run_id: &str,
2928        stream_name: &str,
2929    ) -> Result<WorkflowStreamDescription> {
2930        let response: WorkflowStreamDescriptionResponse = self
2931            .request_json(
2932                reqwest::Method::GET,
2933                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
2934                RequestProtocol::ControlPlane,
2935                Option::<&Value>::None,
2936            )
2937            .await?;
2938        Ok(response.stream)
2939    }
2940
2941    /// Read one bounded page beginning at a zero-based offset.
2942    ///
2943    /// Delivery is at least once: persist `next_offset` only after processing
2944    /// the page. The future is cancellation-safe; dropping it cancels the
2945    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
2946    /// service contract.
2947    pub async fn subscribe_workflow_stream(
2948        &self,
2949        workflow_id: &str,
2950        run_id: &str,
2951        stream_name: &str,
2952        from_offset: u64,
2953        max_items: usize,
2954        wait: Duration,
2955    ) -> Result<WorkflowStreamPage> {
2956        let max_items = max_items.clamp(1, 500);
2957        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
2958        let path = format!(
2959            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
2960            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
2961        );
2962        let response: WorkflowStreamPageResponse = self
2963            .request_json_with_timeout(
2964                reqwest::Method::GET,
2965                &path,
2966                RequestProtocol::ControlPlane,
2967                Option::<&Value>::None,
2968                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
2969            )
2970            .await?;
2971
2972        let items = response
2973            .items
2974            .into_iter()
2975            .map(|raw| {
2976                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
2977                let envelope = raw.get("payload").cloned();
2978                let payload = envelope
2979                    .as_ref()
2980                    .filter(|value| value.get("blob").is_some())
2981                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
2982                    .transpose()?
2983                    .map(AvroValue::into_json)
2984                    .transpose()?;
2985                Ok(WorkflowStreamItem {
2986                    offset,
2987                    payload,
2988                    payload_envelope: envelope,
2989                    payload_reference: raw
2990                        .get("payload_reference")
2991                        .and_then(Value::as_str)
2992                        .map(str::to_string),
2993                    payload_codec: raw
2994                        .get("payload_codec")
2995                        .and_then(Value::as_str)
2996                        .map(str::to_string),
2997                    idempotency_key: raw
2998                        .get("idempotency_key")
2999                        .and_then(Value::as_str)
3000                        .map(str::to_string),
3001                    item_type: raw
3002                        .get("item_type")
3003                        .and_then(Value::as_str)
3004                        .map(str::to_string),
3005                    content_type: raw
3006                        .get("content_type")
3007                        .and_then(Value::as_str)
3008                        .map(str::to_string),
3009                    origin: raw
3010                        .get("origin")
3011                        .and_then(Value::as_str)
3012                        .map(str::to_string),
3013                    origin_reference: raw
3014                        .get("origin_reference")
3015                        .and_then(Value::as_str)
3016                        .map(str::to_string),
3017                    emitted_at: raw
3018                        .get("emitted_at")
3019                        .and_then(Value::as_str)
3020                        .map(str::to_string),
3021                    raw,
3022                })
3023            })
3024            .collect::<Result<Vec<_>>>()?;
3025        Ok(WorkflowStreamPage {
3026            stream: response.stream,
3027            items,
3028            next_offset: response.next_offset,
3029            terminal: response.terminal,
3030        })
3031    }
3032
3033    /// Append inline Avro envelopes or opaque external payload references.
3034    pub async fn append_workflow_stream(
3035        &self,
3036        workflow_id: &str,
3037        run_id: &str,
3038        stream_name: &str,
3039        items: &[WorkflowStreamAppendItem],
3040        max_pending_items: Option<u64>,
3041    ) -> Result<WorkflowStreamAppendResult> {
3042        if items.is_empty() {
3043            return Err(Error::Codec(
3044                "workflow_stream_items_empty: append requires at least one item".to_string(),
3045            ));
3046        }
3047        let mut body = json!({
3048            "items": items
3049                .iter()
3050                .map(|item| item.wire_value(None))
3051                .collect::<Vec<_>>(),
3052        });
3053        if let Some(max_pending_items) = max_pending_items {
3054            if max_pending_items == 0 {
3055                return Err(Error::Codec(
3056                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3057                        .to_string(),
3058                ));
3059            }
3060            body["max_pending_items"] = json!(max_pending_items);
3061        }
3062        let response: WorkflowStreamAppendResponse = self
3063            .request_json(
3064                reqwest::Method::POST,
3065                &format!(
3066                    "{}/items",
3067                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3068                ),
3069                RequestProtocol::ControlPlane,
3070                Some(&body),
3071            )
3072            .await?;
3073        Ok(WorkflowStreamAppendResult {
3074            stream: response.stream,
3075            accepted_offsets: response.accepted_offsets,
3076            accepted: response.accepted,
3077            deduped: response.deduped,
3078        })
3079    }
3080
3081    /// Close a stream, or mark it errored when `error_reason` is supplied.
3082    pub async fn close_workflow_stream(
3083        &self,
3084        workflow_id: &str,
3085        run_id: &str,
3086        stream_name: &str,
3087        error_reason: Option<&str>,
3088        retention_seconds: Option<u64>,
3089    ) -> Result<WorkflowStreamDescription> {
3090        let mut body = json!({});
3091        if let Some(error_reason) = error_reason {
3092            body["error_reason"] = json!(error_reason);
3093        }
3094        if let Some(retention_seconds) = retention_seconds {
3095            if retention_seconds == 0 {
3096                return Err(Error::Codec(
3097                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3098                        .to_string(),
3099                ));
3100            }
3101            body["retention_seconds"] = json!(retention_seconds);
3102        }
3103        let response: WorkflowStreamDescriptionResponse = self
3104            .request_json(
3105                reqwest::Method::POST,
3106                &format!(
3107                    "{}/close",
3108                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3109                ),
3110                RequestProtocol::ControlPlane,
3111                Some(&body),
3112            )
3113            .await?;
3114        Ok(response.stream)
3115    }
3116
3117    pub async fn register_worker(
3118        &self,
3119        worker_id: &str,
3120        task_queue: &str,
3121        supported_workflow_types: Vec<String>,
3122        supported_activity_types: Vec<String>,
3123        max_concurrent_workflow_tasks: usize,
3124        max_concurrent_activity_tasks: usize,
3125    ) -> Result<RegisterWorkerResponse> {
3126        self.register_worker_with_capabilities(
3127            worker_id,
3128            task_queue,
3129            supported_workflow_types,
3130            supported_activity_types,
3131            max_concurrent_workflow_tasks,
3132            max_concurrent_activity_tasks,
3133            Vec::new(),
3134        )
3135        .await
3136    }
3137
3138    /// Register a worker and explicitly advertise additive worker capabilities.
3139    pub async fn register_worker_with_capabilities(
3140        &self,
3141        worker_id: &str,
3142        task_queue: &str,
3143        supported_workflow_types: Vec<String>,
3144        supported_activity_types: Vec<String>,
3145        max_concurrent_workflow_tasks: usize,
3146        max_concurrent_activity_tasks: usize,
3147        capabilities: Vec<String>,
3148    ) -> Result<RegisterWorkerResponse> {
3149        self.register_worker_with_command_contracts(
3150            worker_id,
3151            task_queue,
3152            supported_workflow_types,
3153            supported_activity_types,
3154            max_concurrent_workflow_tasks,
3155            max_concurrent_activity_tasks,
3156            capabilities,
3157            Value::Object(serde_json::Map::new()),
3158        )
3159        .await
3160    }
3161
3162    /// Register a worker and advertise its named query and update handlers.
3163    ///
3164    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3165    /// so a workflow contract with a non-empty or malformed
3166    /// `update_validators` declaration returns
3167    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3168    #[allow(clippy::too_many_arguments)]
3169    pub async fn register_worker_with_command_contracts(
3170        &self,
3171        worker_id: &str,
3172        task_queue: &str,
3173        supported_workflow_types: Vec<String>,
3174        supported_activity_types: Vec<String>,
3175        max_concurrent_workflow_tasks: usize,
3176        max_concurrent_activity_tasks: usize,
3177        capabilities: Vec<String>,
3178        workflow_command_contracts: Value,
3179    ) -> Result<RegisterWorkerResponse> {
3180        if let Some(contracts) = workflow_command_contracts.as_object() {
3181            for (workflow_type, contract) in contracts {
3182                let Some(update_validators) = contract.get("update_validators") else {
3183                    continue;
3184                };
3185                if !update_validators
3186                    .as_array()
3187                    .is_some_and(|validators| validators.is_empty())
3188                {
3189                    return Err(Error::UnsupportedUpdateValidators {
3190                        workflow_type: workflow_type.clone(),
3191                    });
3192                }
3193            }
3194        }
3195
3196        let mut body = json!({
3197            "worker_id": worker_id,
3198            "task_queue": task_queue,
3199            "runtime": "rust",
3200            "sdk_version": SDK_VERSION,
3201            "supported_workflow_types": supported_workflow_types,
3202            "supported_activity_types": supported_activity_types,
3203            "capabilities": capabilities,
3204            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3205            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3206        });
3207        if workflow_command_contracts
3208            .as_object()
3209            .is_some_and(|contracts| !contracts.is_empty())
3210        {
3211            body["workflow_command_contracts"] = workflow_command_contracts;
3212        }
3213
3214        self.request_json(
3215            reqwest::Method::POST,
3216            "/worker/register",
3217            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3218            Some(&body),
3219        )
3220        .await
3221    }
3222
3223    /// Gracefully remove one worker's registration through the worker plane.
3224    ///
3225    /// This operation is separate from operator-facing worker management. It
3226    /// uses worker-protocol authentication and returns the server's lease
3227    /// recovery result.
3228    pub async fn deregister_worker_registration(
3229        &self,
3230        worker_id: &str,
3231    ) -> Result<WorkerDeregistrationEnvelope> {
3232        let path = format!(
3233            "/worker/registrations/{}",
3234            percent_encode_path_segment(worker_id)
3235        );
3236        self.request_json(
3237            reqwest::Method::DELETE,
3238            &path,
3239            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3240            Option::<&Value>::None,
3241        )
3242        .await
3243    }
3244
3245    /// Long-poll for an ephemeral, read-only workflow query task.
3246    pub async fn poll_query_task(
3247        &self,
3248        worker_id: &str,
3249        task_queue: &str,
3250        timeout: Duration,
3251    ) -> Result<Option<QueryTask>> {
3252        Ok(self
3253            .poll_query_task_response(worker_id, task_queue, timeout)
3254            .await?
3255            .task)
3256    }
3257
3258    /// Poll a query task while preserving server stop and drain metadata.
3259    pub async fn poll_query_task_response(
3260        &self,
3261        worker_id: &str,
3262        task_queue: &str,
3263        timeout: Duration,
3264    ) -> Result<PollQueryTaskResponse> {
3265        let poll_request_id = unique_request_id("rust-query-poll");
3266        self.poll_query_task_response_with_request_id(
3267            worker_id,
3268            task_queue,
3269            timeout,
3270            &poll_request_id,
3271            1,
3272        )
3273        .await
3274    }
3275
3276    async fn poll_query_task_response_with_request_id(
3277        &self,
3278        worker_id: &str,
3279        task_queue: &str,
3280        timeout: Duration,
3281        poll_request_id: &str,
3282        transport_retries: usize,
3283    ) -> Result<PollQueryTaskResponse> {
3284        let timeout_seconds = long_poll_timeout_seconds(timeout);
3285        let body = json!({
3286            "worker_id": worker_id,
3287            "task_queue": task_queue,
3288            "poll_request_id": poll_request_id,
3289            "timeout_seconds": timeout_seconds,
3290        });
3291        self.poll_request_json(
3292            "/worker/query-tasks/poll",
3293            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3294            &body,
3295            timeout + Duration::from_secs(5),
3296            transport_retries,
3297        )
3298        .await
3299    }
3300
3301    /// Complete a query task without appending workflow history.
3302    pub async fn complete_query_task<T: Serialize>(
3303        &self,
3304        query_task_id: &str,
3305        lease_owner: &str,
3306        query_task_attempt: u64,
3307        result: T,
3308        codec: &str,
3309    ) -> Result<Value> {
3310        let typed_result = AvroValue::from_serialize(&result)?;
3311        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3312        self.complete_query_task_with_envelope(
3313            query_task_id,
3314            lease_owner,
3315            query_task_attempt,
3316            typed_result.into_json()?,
3317            result_envelope,
3318        )
3319        .await
3320    }
3321
3322    async fn complete_query_task_with_envelope(
3323        &self,
3324        query_task_id: &str,
3325        lease_owner: &str,
3326        query_task_attempt: u64,
3327        result: Value,
3328        result_envelope: Value,
3329    ) -> Result<Value> {
3330        let body = json!({
3331            "lease_owner": lease_owner,
3332            "query_task_attempt": query_task_attempt,
3333            "result": result,
3334            "result_envelope": result_envelope,
3335        });
3336        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3337        let response = self
3338            .request_json(
3339                reqwest::Method::POST,
3340                &path,
3341                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3342                Some(&body),
3343            )
3344            .await;
3345        query_task_response(response)
3346    }
3347
3348    /// Report a stable machine-readable query-task failure.
3349    pub async fn fail_query_task(
3350        &self,
3351        query_task_id: &str,
3352        lease_owner: &str,
3353        query_task_attempt: u64,
3354        message: impl Into<String>,
3355        reason: impl Into<String>,
3356        failure_type: impl Into<String>,
3357    ) -> Result<Value> {
3358        let body = json!({
3359            "lease_owner": lease_owner,
3360            "query_task_attempt": query_task_attempt,
3361            "failure": {
3362                "message": message.into(),
3363                "reason": reason.into(),
3364                "type": failure_type.into(),
3365            }
3366        });
3367        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3368        let response = self
3369            .request_json(
3370                reqwest::Method::POST,
3371                &path,
3372                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3373                Some(&body),
3374            )
3375            .await;
3376        query_task_response(response)
3377    }
3378
3379    pub async fn heartbeat_worker(
3380        &self,
3381        worker_id: &str,
3382        workflow_available: usize,
3383        activity_available: usize,
3384    ) -> Result<Value> {
3385        let body = json!({
3386            "worker_id": worker_id,
3387            "task_slots": {
3388                "workflow_available": workflow_available,
3389                "activity_available": activity_available
3390            },
3391            "process_metrics": {
3392                "process_id": std::process::id(),
3393                "process_uptime_seconds": 0
3394            }
3395        });
3396
3397        self.request_json(
3398            reqwest::Method::POST,
3399            "/worker/heartbeat",
3400            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3401            Some(&body),
3402        )
3403        .await
3404    }
3405
3406    pub async fn poll_workflow_task(
3407        &self,
3408        worker_id: &str,
3409        task_queue: &str,
3410        timeout: Duration,
3411    ) -> Result<Option<WorkflowTask>> {
3412        Ok(self
3413            .poll_workflow_task_response(worker_id, task_queue, timeout)
3414            .await?
3415            .task)
3416    }
3417
3418    pub async fn poll_workflow_task_response(
3419        &self,
3420        worker_id: &str,
3421        task_queue: &str,
3422        timeout: Duration,
3423    ) -> Result<PollWorkflowTaskResponse> {
3424        let poll_request_id = unique_request_id("rust-workflow-poll");
3425        self.poll_workflow_task_response_with_request_id(
3426            worker_id,
3427            task_queue,
3428            timeout,
3429            &poll_request_id,
3430            1,
3431        )
3432        .await
3433    }
3434
3435    async fn poll_workflow_task_response_with_request_id(
3436        &self,
3437        worker_id: &str,
3438        task_queue: &str,
3439        timeout: Duration,
3440        poll_request_id: &str,
3441        transport_retries: usize,
3442    ) -> Result<PollWorkflowTaskResponse> {
3443        let body = json!({
3444            "worker_id": worker_id,
3445            "task_queue": task_queue,
3446            "poll_request_id": poll_request_id,
3447            "timeout_seconds": long_poll_timeout_seconds(timeout),
3448        });
3449        let mut data: PollWorkflowTaskResponse = self
3450            .poll_request_json(
3451                "/worker/workflow-tasks/poll",
3452                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3453                &body,
3454                timeout + Duration::from_secs(5),
3455                transport_retries,
3456            )
3457            .await?;
3458
3459        if let Some(task) = data.task.as_mut() {
3460            self.fetch_remaining_workflow_history(worker_id, task)
3461                .await?;
3462        }
3463
3464        Ok(data)
3465    }
3466
3467    async fn fetch_remaining_workflow_history(
3468        &self,
3469        worker_id: &str,
3470        task: &mut WorkflowTask,
3471    ) -> Result<()> {
3472        let mut next_token = task.next_history_page_token.clone();
3473
3474        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3475            let lease_owner = task
3476                .lease_owner
3477                .clone()
3478                .unwrap_or_else(|| worker_id.to_string());
3479            let page = self
3480                .workflow_task_history_page(
3481                    &task.task_id,
3482                    &lease_owner,
3483                    task.workflow_task_attempt,
3484                    &token,
3485                )
3486                .await?;
3487
3488            task.append_history_page(page);
3489
3490            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3491                return Err(Error::Codec(
3492                    "workflow history pagination returned the same page token".to_string(),
3493                ));
3494            }
3495
3496            next_token = task.next_history_page_token.clone();
3497        }
3498
3499        Ok(())
3500    }
3501
3502    async fn workflow_task_history_page(
3503        &self,
3504        task_id: &str,
3505        lease_owner: &str,
3506        workflow_task_attempt: u64,
3507        next_history_page_token: &str,
3508    ) -> Result<WorkflowTaskHistoryPage> {
3509        let body = json!({
3510            "lease_owner": lease_owner,
3511            "workflow_task_attempt": workflow_task_attempt,
3512            "next_history_page_token": next_history_page_token
3513        });
3514        let path = format!("/worker/workflow-tasks/{task_id}/history");
3515
3516        self.request_json(
3517            reqwest::Method::POST,
3518            &path,
3519            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3520            Some(&body),
3521        )
3522        .await
3523    }
3524
3525    pub async fn complete_workflow_task(
3526        &self,
3527        task_id: &str,
3528        lease_owner: &str,
3529        workflow_task_attempt: u64,
3530        commands: Vec<Value>,
3531    ) -> Result<Value> {
3532        validate_workflow_task_commands(&commands)?;
3533        let protocol_version = workflow_completion_protocol_version(&commands);
3534        let body = json!({
3535            "lease_owner": lease_owner,
3536            "workflow_task_attempt": workflow_task_attempt,
3537            "commands": commands
3538        });
3539        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3540        self.request_json(
3541            reqwest::Method::POST,
3542            &path,
3543            RequestProtocol::Worker(protocol_version),
3544            Some(&body),
3545        )
3546        .await
3547    }
3548
3549    pub async fn fail_workflow_task(
3550        &self,
3551        task_id: &str,
3552        lease_owner: &str,
3553        workflow_task_attempt: u64,
3554        message: impl Into<String>,
3555    ) -> Result<Value> {
3556        self.fail_workflow_task_with_type(
3557            task_id,
3558            lease_owner,
3559            workflow_task_attempt,
3560            message,
3561            "RustWorkflowTaskFailure",
3562        )
3563        .await
3564    }
3565
3566    async fn fail_workflow_task_with_type(
3567        &self,
3568        task_id: &str,
3569        lease_owner: &str,
3570        workflow_task_attempt: u64,
3571        message: impl Into<String>,
3572        failure_type: &str,
3573    ) -> Result<Value> {
3574        let body = json!({
3575            "lease_owner": lease_owner,
3576            "workflow_task_attempt": workflow_task_attempt,
3577            "failure": {
3578                "message": message.into(),
3579                "type": failure_type
3580            }
3581        });
3582        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3583        self.request_json(
3584            reqwest::Method::POST,
3585            &path,
3586            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3587            Some(&body),
3588        )
3589        .await
3590    }
3591
3592    pub async fn poll_activity_task(
3593        &self,
3594        worker_id: &str,
3595        task_queue: &str,
3596        timeout: Duration,
3597    ) -> Result<Option<ActivityTask>> {
3598        Ok(self
3599            .poll_activity_task_response(worker_id, task_queue, timeout)
3600            .await?
3601            .task)
3602    }
3603
3604    /// Poll an activity task while preserving server stop and drain metadata.
3605    pub async fn poll_activity_task_response(
3606        &self,
3607        worker_id: &str,
3608        task_queue: &str,
3609        timeout: Duration,
3610    ) -> Result<PollActivityTaskResponse> {
3611        let poll_request_id = unique_request_id("rust-activity-poll");
3612        self.poll_activity_task_response_with_request_id(
3613            worker_id,
3614            task_queue,
3615            timeout,
3616            &poll_request_id,
3617            1,
3618        )
3619        .await
3620    }
3621
3622    async fn poll_activity_task_response_with_request_id(
3623        &self,
3624        worker_id: &str,
3625        task_queue: &str,
3626        timeout: Duration,
3627        poll_request_id: &str,
3628        transport_retries: usize,
3629    ) -> Result<PollActivityTaskResponse> {
3630        let body = json!({
3631            "worker_id": worker_id,
3632            "task_queue": task_queue,
3633            "poll_request_id": poll_request_id,
3634            "timeout_seconds": long_poll_timeout_seconds(timeout),
3635        });
3636        let data: PollActivityTaskResponse = self
3637            .poll_request_json(
3638                "/worker/activity-tasks/poll",
3639                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3640                &body,
3641                timeout + Duration::from_secs(5),
3642                transport_retries,
3643            )
3644            .await?;
3645        Ok(data)
3646    }
3647
3648    pub async fn complete_activity_task<T: Serialize>(
3649        &self,
3650        task_id: &str,
3651        activity_attempt_id: &str,
3652        lease_owner: &str,
3653        result: T,
3654        codec: &str,
3655    ) -> Result<Value> {
3656        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3657        let body = json!({
3658            "activity_attempt_id": activity_attempt_id,
3659            "lease_owner": lease_owner,
3660            "result": result
3661        });
3662        let path = format!("/worker/activity-tasks/{task_id}/complete");
3663        activity_task_response(
3664            self.request_json(
3665                reqwest::Method::POST,
3666                &path,
3667                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3668                Some(&body),
3669            )
3670            .await,
3671            "complete",
3672            task_id,
3673            activity_attempt_id,
3674        )
3675    }
3676
3677    pub async fn fail_activity_task(
3678        &self,
3679        task_id: &str,
3680        activity_attempt_id: &str,
3681        lease_owner: &str,
3682        message: impl Into<String>,
3683        non_retryable: bool,
3684    ) -> Result<Value> {
3685        let body = json!({
3686            "activity_attempt_id": activity_attempt_id,
3687            "lease_owner": lease_owner,
3688            "failure": {
3689                "message": message.into(),
3690                "type": "RustActivityFailure",
3691                "non_retryable": non_retryable
3692            }
3693        });
3694        let path = format!("/worker/activity-tasks/{task_id}/fail");
3695        activity_task_response(
3696            self.request_json(
3697                reqwest::Method::POST,
3698                &path,
3699                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3700                Some(&body),
3701            )
3702            .await,
3703            "fail",
3704            task_id,
3705            activity_attempt_id,
3706        )
3707    }
3708
3709    pub async fn heartbeat_activity_task<T: Serialize>(
3710        &self,
3711        task_id: &str,
3712        activity_attempt_id: &str,
3713        lease_owner: &str,
3714        details: T,
3715    ) -> Result<ActivityHeartbeatResponse> {
3716        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3717        let body = json!({
3718            "activity_attempt_id": activity_attempt_id,
3719            "lease_owner": lease_owner,
3720            "details": details
3721        });
3722        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
3723        activity_task_response(
3724            self.request_json(
3725                reqwest::Method::POST,
3726                &path,
3727                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3728                Some(&body),
3729            )
3730            .await,
3731            "heartbeat",
3732            task_id,
3733            activity_attempt_id,
3734        )
3735    }
3736
3737    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
3738        &self,
3739        method: reqwest::Method,
3740        path: &str,
3741        protocol: RequestProtocol,
3742        body: Option<&B>,
3743    ) -> Result<T> {
3744        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
3745            .await
3746    }
3747
3748    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
3749        &self,
3750        method: reqwest::Method,
3751        path: &str,
3752        protocol: RequestProtocol,
3753        body: Option<&B>,
3754        timeout: Duration,
3755    ) -> Result<T> {
3756        let auth_token = self.auth_token(protocol)?;
3757        let mut request = self
3758            .http
3759            .request(method, format!("{}/api{}", self.base_url, path))
3760            .timeout(timeout)
3761            .header(reqwest::header::ACCEPT, "application/json")
3762            .header(reqwest::header::CONTENT_TYPE, "application/json")
3763            .header("X-Namespace", &self.namespace);
3764
3765        match protocol {
3766            RequestProtocol::Worker(version) => {
3767                request = request.header("X-Durable-Workflow-Protocol-Version", version);
3768            }
3769            RequestProtocol::ControlPlane => {
3770                request = request.header(
3771                    "X-Durable-Workflow-Control-Plane-Version",
3772                    CONTROL_PLANE_VERSION,
3773                );
3774            }
3775        }
3776
3777        if let Some(token) = auth_token {
3778            request = request.bearer_auth(token);
3779        }
3780
3781        if let Some(body) = body {
3782            request = request.json(body);
3783        }
3784
3785        let response = request.send().await?;
3786        let status = response.status();
3787        let bytes = response.bytes().await?;
3788
3789        if !status.is_success() {
3790            let body = String::from_utf8_lossy(&bytes).to_string();
3791            if let Some(protocol) = protocol_failure(status, &body) {
3792                return Err(Error::Protocol(protocol));
3793            }
3794            return Err(Error::Http { status, body });
3795        }
3796
3797        if bytes.is_empty() {
3798            return Ok(serde_json::from_value(Value::Null)?);
3799        }
3800
3801        Ok(serde_json::from_slice(&bytes)?)
3802    }
3803
3804    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
3805        &self,
3806        path: &str,
3807        protocol: RequestProtocol,
3808        body: &B,
3809        timeout: Duration,
3810        max_retries: usize,
3811    ) -> Result<T> {
3812        let mut retries = 0;
3813
3814        loop {
3815            let response = self
3816                .request_json_with_timeout(
3817                    reqwest::Method::POST,
3818                    path,
3819                    protocol,
3820                    Some(body),
3821                    timeout,
3822                )
3823                .await;
3824
3825            match response {
3826                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
3827                response => return worker_poll_response(response),
3828            }
3829        }
3830    }
3831
3832    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
3833        match protocol {
3834            RequestProtocol::Worker(_) => {
3835                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
3836                    return Ok(Some(token));
3837                }
3838                if self.control_token.is_some() {
3839                    return Err(Error::MissingRoleCredentials {
3840                        role: "worker",
3841                        opposite_role: "control",
3842                    });
3843                }
3844                Ok(None)
3845            }
3846            RequestProtocol::ControlPlane => {
3847                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
3848                    return Ok(Some(token));
3849                }
3850                if self.worker_token.is_some() {
3851                    return Err(Error::MissingRoleCredentials {
3852                        role: "control",
3853                        opposite_role: "worker",
3854                    });
3855                }
3856                Ok(None)
3857            }
3858        }
3859    }
3860}
3861
3862fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
3863    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
3864    let reason = body
3865        .get("reason")
3866        .and_then(Value::as_str)
3867        .unwrap_or("query_rejected")
3868        .to_string();
3869    let message = body
3870        .get("message")
3871        .or_else(|| body.get("error"))
3872        .and_then(Value::as_str)
3873        .unwrap_or("workflow query was rejected")
3874        .to_string();
3875
3876    QueryFailure {
3877        status: status.as_u16(),
3878        reason,
3879        message,
3880        body,
3881    }
3882}
3883
3884fn workflow_command_result(
3885    command: WorkflowCommandKind,
3886    data: Value,
3887    workflow_id: &str,
3888    run_id: Option<&str>,
3889) -> WorkflowCommandResult {
3890    WorkflowCommandResult {
3891        command,
3892        workflow_id: data
3893            .get("workflow_id")
3894            .and_then(Value::as_str)
3895            .unwrap_or(workflow_id)
3896            .to_string(),
3897        run_id: data
3898            .get("run_id")
3899            .and_then(Value::as_str)
3900            .or(run_id)
3901            .map(str::to_string),
3902        outcome: data
3903            .get("outcome")
3904            .and_then(Value::as_str)
3905            .map(str::to_string),
3906        reason: data
3907            .get("reason")
3908            .and_then(Value::as_str)
3909            .map(str::to_string),
3910        command_status: data
3911            .get("command_status")
3912            .and_then(Value::as_str)
3913            .map(str::to_string),
3914        raw: data,
3915    }
3916}
3917
3918fn workflow_command_rejection(
3919    command: WorkflowCommandKind,
3920    status: reqwest::StatusCode,
3921    raw_body: String,
3922    workflow_id: &str,
3923    run_id: Option<&str>,
3924) -> WorkflowCommandRejection {
3925    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
3926    WorkflowCommandRejection {
3927        command,
3928        status: status.as_u16(),
3929        reason: body
3930            .get("reason")
3931            .and_then(Value::as_str)
3932            .unwrap_or("workflow_command_rejected")
3933            .to_string(),
3934        message: body
3935            .get("message")
3936            .or_else(|| body.get("error"))
3937            .and_then(Value::as_str)
3938            .unwrap_or("workflow lifecycle command was rejected")
3939            .to_string(),
3940        workflow_id: body
3941            .get("workflow_id")
3942            .and_then(Value::as_str)
3943            .unwrap_or(workflow_id)
3944            .to_string(),
3945        run_id: body
3946            .get("run_id")
3947            .and_then(Value::as_str)
3948            .or(run_id)
3949            .map(str::to_string),
3950        target_scope: body
3951            .get("target_scope")
3952            .and_then(Value::as_str)
3953            .map(str::to_string),
3954        body,
3955    }
3956}
3957
3958fn query_task_response(response: Result<Value>) -> Result<Value> {
3959    match response {
3960        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
3961        response => response,
3962    }
3963}
3964
3965fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
3966    match response {
3967        Err(Error::Http { status, body })
3968            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
3969        {
3970            Ok(serde_json::from_str(&body)?)
3971        }
3972        response => response,
3973    }
3974}
3975
3976fn worker_poll_body_is_stop(body: &str) -> bool {
3977    serde_json::from_str::<Value>(body)
3978        .ok()
3979        .is_some_and(|body| {
3980            worker_poll_is_stop(
3981                body.get("poll_status").and_then(Value::as_str),
3982                body.get("reason").and_then(Value::as_str),
3983            )
3984        })
3985}
3986
3987fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
3988    matches!(poll_status, Some("draining" | "stopped"))
3989        || matches!(reason, Some("worker_draining" | "worker_stopped"))
3990}
3991
3992fn query_task_rejection_is_final(error: &Error) -> bool {
3993    matches!(
3994        error,
3995        Error::QueryFailed(failure)
3996            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
3997    )
3998}
3999
4000fn activity_task_response<T>(
4001    response: Result<T>,
4002    operation: &str,
4003    task_id: &str,
4004    activity_attempt_id: &str,
4005) -> Result<T> {
4006    match response {
4007        Err(Error::Http { status, body }) => {
4008            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4009            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4010                operation: operation.to_string(),
4011                status: status.as_u16(),
4012                reason: body
4013                    .get("reason")
4014                    .and_then(Value::as_str)
4015                    .unwrap_or("activity_task_rejected")
4016                    .to_string(),
4017                task_id: body
4018                    .get("task_id")
4019                    .and_then(Value::as_str)
4020                    .unwrap_or(task_id)
4021                    .to_string(),
4022                activity_attempt_id: body
4023                    .get("activity_attempt_id")
4024                    .and_then(Value::as_str)
4025                    .unwrap_or(activity_attempt_id)
4026                    .to_string(),
4027                cancel_requested: body
4028                    .get("cancel_requested")
4029                    .and_then(Value::as_bool)
4030                    .unwrap_or(false),
4031                can_continue: body.get("can_continue").and_then(Value::as_bool),
4032                run_closed_reason: body
4033                    .get("run_closed_reason")
4034                    .and_then(Value::as_str)
4035                    .map(str::to_string),
4036                body,
4037            }))
4038        }
4039        response => response,
4040    }
4041}
4042
4043fn activity_task_rejection_is_final(error: &Error) -> bool {
4044    matches!(
4045        error,
4046        Error::ActivityTaskRejected(rejection)
4047            if matches!(
4048                rejection.reason.as_str(),
4049                "run_cancelled"
4050                    | "run_terminated"
4051                    | "attempt_closed"
4052                    | "stale_attempt"
4053                    | "activity_cancelled"
4054                    | "task_cancelled"
4055                    | "run_closed"
4056                    | "activity_not_running"
4057                    | "attempt_not_found"
4058            )
4059    )
4060}
4061
4062fn workflow_task_completion_is_terminal_timeout(
4063    error: &Error,
4064    task_id: &str,
4065    workflow_task_attempt: u64,
4066    run_id: Option<&str>,
4067) -> bool {
4068    let Error::Http { status, body } = error else {
4069        return false;
4070    };
4071    if *status != reqwest::StatusCode::CONFLICT {
4072        return false;
4073    }
4074
4075    let Some(run_id) = run_id else {
4076        return false;
4077    };
4078    let Ok(body) = serde_json::from_str::<Value>(body) else {
4079        return false;
4080    };
4081
4082    body.get("recorded").and_then(Value::as_bool) == Some(false)
4083        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4084        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4085        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4086        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4087        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4088}
4089
4090fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4091    let body: Value = serde_json::from_str(raw_body).ok()?;
4092    let reason = body.get("reason")?.as_str()?;
4093    if !matches!(
4094        reason,
4095        "missing_protocol_version"
4096            | "unsupported_protocol_version"
4097            | "missing_control_plane_version"
4098            | "unsupported_control_plane_version"
4099    ) {
4100        return None;
4101    }
4102
4103    Some(ProtocolFailure {
4104        status: status.as_u16(),
4105        reason: reason.to_string(),
4106        message: body
4107            .get("message")
4108            .or_else(|| body.get("error"))
4109            .and_then(Value::as_str)
4110            .unwrap_or("protocol version rejected")
4111            .to_string(),
4112        supported_version: body
4113            .get("supported_version")
4114            .and_then(Value::as_str)
4115            .map(str::to_string),
4116        requested_version: body
4117            .get("requested_version")
4118            .and_then(Value::as_str)
4119            .map(str::to_string),
4120        body,
4121    })
4122}
4123
4124fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4125    timeout
4126        .as_secs()
4127        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4128        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4129}
4130
4131fn worker_operation_is_retryable(error: &Error) -> bool {
4132    match error {
4133        Error::Transport(error) => {
4134            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4135        }
4136        Error::Http { status, .. } => {
4137            matches!(
4138                *status,
4139                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4140            ) || status.is_server_error()
4141        }
4142        _ => false,
4143    }
4144}
4145
4146fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4147    let exponent = retry.saturating_sub(1).min(31) as u32;
4148    policy
4149        .initial_backoff
4150        .saturating_mul(1_u32 << exponent)
4151        .min(policy.max_backoff)
4152}
4153
4154#[derive(Debug)]
4155pub struct ClientBuilder {
4156    base_url: String,
4157    token: Option<String>,
4158    control_token: Option<String>,
4159    worker_token: Option<String>,
4160    namespace: String,
4161    timeout: Duration,
4162}
4163
4164impl ClientBuilder {
4165    pub fn token(mut self, token: Option<String>) -> Self {
4166        self.token = token;
4167        self
4168    }
4169
4170    pub fn control_token(mut self, token: Option<String>) -> Self {
4171        self.control_token = token;
4172        self
4173    }
4174
4175    pub fn worker_token(mut self, token: Option<String>) -> Self {
4176        self.worker_token = token;
4177        self
4178    }
4179
4180    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4181        self.namespace = namespace.into();
4182        self
4183    }
4184
4185    pub fn timeout(mut self, timeout: Duration) -> Self {
4186        self.timeout = timeout;
4187        self
4188    }
4189
4190    pub fn build(self) -> Result<Client> {
4191        let base_url = self.base_url.trim_end_matches('/').to_string();
4192        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4193            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4194            .unwrap_or_else(|_| base_url.ends_with("/api"));
4195
4196        if has_sdk_api_suffix {
4197            return Err(Error::InvalidBaseUrl);
4198        }
4199
4200        Ok(Client {
4201            http: reqwest::Client::builder().timeout(self.timeout).build()?,
4202            base_url,
4203            token: self.token,
4204            control_token: self.control_token,
4205            worker_token: self.worker_token,
4206            namespace: self.namespace,
4207        })
4208    }
4209}
4210
4211#[derive(Clone, Debug)]
4212pub struct WorkflowHandle {
4213    client: Client,
4214    pub workflow_id: String,
4215    pub run_id: Option<String>,
4216    pub workflow_type: String,
4217}
4218
4219impl WorkflowHandle {
4220    /// Describe whichever run is current for this stable workflow instance.
4221    pub async fn describe(&self) -> Result<WorkflowDescription> {
4222        self.client.describe_workflow(&self.workflow_id).await
4223    }
4224
4225    /// Describe the run identity originally selected by this handle.
4226    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4227        let run_id = self.run_id.as_deref().ok_or_else(|| {
4228            Error::Codec("run_id is required for selected-run description".to_string())
4229        })?;
4230        self.client
4231            .describe_workflow_run(&self.workflow_id, run_id)
4232            .await
4233    }
4234
4235    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4236        self.client
4237            .signal_workflow(&self.workflow_id, signal_name, input)
4238            .await
4239    }
4240
4241    /// Signal only if this handle's selected run is still current.
4242    pub async fn signal_selected_run<T: Serialize>(
4243        &self,
4244        signal_name: &str,
4245        input: T,
4246    ) -> Result<Value> {
4247        let run_id = self.run_id.as_deref().ok_or_else(|| {
4248            Error::Codec("run_id is required for selected-run signaling".to_string())
4249        })?;
4250        self.client
4251            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4252            .await
4253    }
4254
4255    /// Request cooperative cancellation of whichever run is current.
4256    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4257        self.client
4258            .cancel_workflow(&self.workflow_id, options)
4259            .await
4260    }
4261
4262    /// Request cancellation only if this handle's selected run is still current.
4263    pub async fn cancel_selected_run(
4264        &self,
4265        options: WorkflowCommandOptions,
4266    ) -> Result<WorkflowCommandResult> {
4267        let run_id = self.run_id.as_deref().ok_or_else(|| {
4268            Error::Codec("run_id is required for selected-run cancellation".to_string())
4269        })?;
4270        self.client
4271            .cancel_workflow_run(&self.workflow_id, run_id, options)
4272            .await
4273    }
4274
4275    /// Forcefully terminate whichever run is current.
4276    pub async fn terminate(
4277        &self,
4278        options: WorkflowCommandOptions,
4279    ) -> Result<WorkflowCommandResult> {
4280        self.client
4281            .terminate_workflow(&self.workflow_id, options)
4282            .await
4283    }
4284
4285    /// Terminate only if this handle's selected run is still current.
4286    pub async fn terminate_selected_run(
4287        &self,
4288        options: WorkflowCommandOptions,
4289    ) -> Result<WorkflowCommandResult> {
4290        let run_id = self.run_id.as_deref().ok_or_else(|| {
4291            Error::Codec("run_id is required for selected-run termination".to_string())
4292        })?;
4293        self.client
4294            .terminate_workflow_run(&self.workflow_id, run_id, options)
4295            .await
4296    }
4297
4298    /// Execute a named, read-only query against this workflow.
4299    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4300        self.client
4301            .query_workflow(&self.workflow_id, query_name, input)
4302            .await
4303    }
4304
4305    pub async fn query_avro_value<T: Serialize>(
4306        &self,
4307        query_name: &str,
4308        input: T,
4309    ) -> Result<AvroValue> {
4310        self.client
4311            .query_workflow_avro_value(&self.workflow_id, query_name, input)
4312            .await
4313    }
4314
4315    pub async fn update<T: Serialize>(
4316        &self,
4317        update_name: &str,
4318        input: T,
4319        request_id: Option<&str>,
4320    ) -> Result<Value> {
4321        self.client
4322            .update_workflow(&self.workflow_id, update_name, input, request_id)
4323            .await
4324    }
4325
4326    pub async fn update_avro_value<T: Serialize>(
4327        &self,
4328        update_name: &str,
4329        input: T,
4330        request_id: Option<&str>,
4331    ) -> Result<AvroValue> {
4332        self.client
4333            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4334            .await
4335    }
4336
4337    /// Query only if this handle's selected run is still current.
4338    pub async fn query_selected_run<T: Serialize>(
4339        &self,
4340        query_name: &str,
4341        input: T,
4342    ) -> Result<Value> {
4343        let run_id = self
4344            .run_id
4345            .as_deref()
4346            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4347        self.client
4348            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4349            .await
4350    }
4351
4352    /// Await the final terminal outcome of the current continue-as-new chain.
4353    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4354        self.result_target(options, None).await
4355    }
4356
4357    /// Await the final result without projecting Avro bytes through JSON.
4358    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4359        self.result_avro_value_target(options, None).await
4360    }
4361
4362    /// Await the final result and decode it into a Serde application type.
4363    pub async fn result_typed<T: DeserializeOwned>(
4364        &self,
4365        options: WorkflowResultOptions,
4366    ) -> Result<T> {
4367        let result = self.result_avro_value(options).await?;
4368        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4369    }
4370
4371    /// Await only the run identity originally selected by this handle.
4372    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4373        let run_id = self.run_id.as_deref().ok_or_else(|| {
4374            Error::Codec("run_id is required for selected-run result".to_string())
4375        })?;
4376        self.result_target(options, Some(run_id)).await
4377    }
4378
4379    /// Await the selected run's result on the lossless Avro Value surface.
4380    pub async fn result_selected_run_avro_value(
4381        &self,
4382        options: WorkflowResultOptions,
4383    ) -> Result<AvroValue> {
4384        let run_id = self.run_id.as_deref().ok_or_else(|| {
4385            Error::Codec("run_id is required for selected-run result".to_string())
4386        })?;
4387        self.result_avro_value_target(options, Some(run_id)).await
4388    }
4389
4390    /// Await the selected run and decode its result into a Serde type.
4391    pub async fn result_selected_run_typed<T: DeserializeOwned>(
4392        &self,
4393        options: WorkflowResultOptions,
4394    ) -> Result<T> {
4395        let result = self.result_selected_run_avro_value(options).await?;
4396        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4397    }
4398
4399    async fn result_avro_value_target(
4400        &self,
4401        options: WorkflowResultOptions,
4402        selected_run_id: Option<&str>,
4403    ) -> Result<AvroValue> {
4404        let started = Instant::now();
4405
4406        loop {
4407            let description = match selected_run_id {
4408                Some(run_id) => {
4409                    self.client
4410                        .describe_workflow_run(&self.workflow_id, run_id)
4411                        .await?
4412                }
4413                None => self.describe().await?,
4414            };
4415            if description.is_completed() {
4416                return description.output_avro_value.ok_or_else(|| {
4417                    Error::Codec(
4418                        "missing_payload_envelope: typed workflow result requires output_envelope"
4419                            .to_string(),
4420                    )
4421                });
4422            }
4423            if description.is_terminal() {
4424                let outcome =
4425                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4426                return Err(match outcome.kind {
4427                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4428                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4429                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4430                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4431                });
4432            }
4433            if started.elapsed() >= options.timeout {
4434                return Err(Error::Timeout);
4435            }
4436            tokio::time::sleep(options.poll_interval).await;
4437        }
4438    }
4439
4440    async fn result_target(
4441        &self,
4442        options: WorkflowResultOptions,
4443        selected_run_id: Option<&str>,
4444    ) -> Result<Value> {
4445        let started = Instant::now();
4446
4447        loop {
4448            let description = match selected_run_id {
4449                Some(run_id) => {
4450                    self.client
4451                        .describe_workflow_run(&self.workflow_id, run_id)
4452                        .await?
4453                }
4454                None => self.describe().await?,
4455            };
4456            if description.is_completed() {
4457                return Ok(description.output.unwrap_or(Value::Null));
4458            }
4459
4460            if description.is_terminal() {
4461                let outcome =
4462                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4463                return Err(match outcome.kind {
4464                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4465                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4466                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4467                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4468                });
4469            }
4470
4471            if started.elapsed() >= options.timeout {
4472                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4473                    kind: WorkflowTerminalKind::TimedOut,
4474                    workflow_id: description
4475                        .workflow_id
4476                        .clone()
4477                        .unwrap_or_else(|| self.workflow_id.clone()),
4478                    run_id: description
4479                        .run_id
4480                        .clone()
4481                        .or_else(|| selected_run_id.map(str::to_string)),
4482                    reason: "result_wait_timeout".to_string(),
4483                    failure_category: Some("client_timeout".to_string()),
4484                    failure_id: None,
4485                    exception_type: None,
4486                    exception_class: None,
4487                    non_retryable: None,
4488                    message: Some(format!(
4489                        "workflow result was not terminal within {:?}",
4490                        options.timeout
4491                    )),
4492                    exception: None,
4493                    raw: description.raw_value(),
4494                }));
4495            }
4496
4497            tokio::time::sleep(options.poll_interval).await;
4498        }
4499    }
4500}
4501
4502#[derive(Clone, Copy, Debug)]
4503pub struct WorkflowResultOptions {
4504    pub poll_interval: Duration,
4505    pub timeout: Duration,
4506}
4507
4508impl Default for WorkflowResultOptions {
4509    fn default() -> Self {
4510        Self {
4511            poll_interval: Duration::from_millis(500),
4512            timeout: Duration::from_secs(30),
4513        }
4514    }
4515}
4516
4517#[derive(Clone, Debug, Deserialize)]
4518pub struct WorkflowDescription {
4519    pub workflow_id: Option<String>,
4520    pub run_id: Option<String>,
4521    pub workflow_type: Option<String>,
4522    pub status: Option<String>,
4523    #[serde(default)]
4524    pub closed_reason: Option<String>,
4525    #[serde(default)]
4526    pub error: Option<String>,
4527    #[serde(default)]
4528    pub failure: Option<Value>,
4529    #[serde(default)]
4530    pub exception: Option<Value>,
4531    #[serde(default)]
4532    pub failures: Vec<Value>,
4533    #[serde(default)]
4534    pub output: Option<Value>,
4535    #[serde(default)]
4536    pub output_envelope: Option<Value>,
4537    #[serde(skip)]
4538    pub output_avro_value: Option<AvroValue>,
4539    #[serde(flatten)]
4540    pub raw: HashMap<String, Value>,
4541}
4542
4543/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
4544#[derive(Clone, Debug, Deserialize)]
4545pub struct WorkflowStreamDescription {
4546    pub stream_name: String,
4547    pub status: String,
4548    pub last_offset: i64,
4549    pub total_items: u64,
4550    pub pending_items: u64,
4551    #[serde(default)]
4552    pub opened_at: Option<String>,
4553    #[serde(default)]
4554    pub last_appended_at: Option<String>,
4555    #[serde(default)]
4556    pub closed_at: Option<String>,
4557    #[serde(default)]
4558    pub error_reason: Option<String>,
4559    #[serde(default)]
4560    pub retention_seconds: Option<u64>,
4561    #[serde(flatten)]
4562    pub raw: HashMap<String, Value>,
4563}
4564
4565impl WorkflowStreamDescription {
4566    pub fn is_terminal(&self) -> bool {
4567        matches!(self.status.as_str(), "closed" | "errored")
4568    }
4569}
4570
4571/// One item for direct or replay-safe append.
4572#[derive(Clone, Debug, Default)]
4573pub struct WorkflowStreamAppendItem {
4574    pub payload_envelope: Option<Value>,
4575    pub payload_reference: Option<String>,
4576    pub item_type: Option<String>,
4577    pub content_type: Option<String>,
4578    pub idempotency_key: Option<String>,
4579}
4580
4581impl WorkflowStreamAppendItem {
4582    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
4583    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
4584        let value = AvroValue::from_serialize(&payload)?;
4585        Ok(Self {
4586            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
4587            ..Self::default()
4588        })
4589    }
4590
4591    /// Preserve an external payload URI as an opaque service-contract reference.
4592    pub fn from_reference(reference: impl Into<String>) -> Self {
4593        Self {
4594            payload_reference: Some(reference.into()),
4595            ..Self::default()
4596        }
4597    }
4598
4599    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
4600        self.item_type = Some(item_type.into());
4601        self
4602    }
4603
4604    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
4605        self.content_type = Some(content_type.into());
4606        self
4607    }
4608
4609    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
4610        self.idempotency_key = Some(idempotency_key.into());
4611        self
4612    }
4613
4614    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
4615        let mut item = serde_json::Map::new();
4616        if let Some(payload) = &self.payload_envelope {
4617            item.insert("payload".to_string(), payload.clone());
4618            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
4619        }
4620        if let Some(reference) = &self.payload_reference {
4621            item.insert("payload_reference".to_string(), json!(reference));
4622        }
4623        if let Some(item_type) = &self.item_type {
4624            item.insert("item_type".to_string(), json!(item_type));
4625        }
4626        if let Some(content_type) = &self.content_type {
4627            item.insert("content_type".to_string(), json!(content_type));
4628        }
4629        if let Some(key) = derived_idempotency_key
4630            .as_ref()
4631            .or(self.idempotency_key.as_ref())
4632        {
4633            item.insert("idempotency_key".to_string(), json!(key));
4634        }
4635        Value::Object(item)
4636    }
4637}
4638
4639/// One durable item at its stable zero-based offset.
4640#[derive(Clone, Debug)]
4641pub struct WorkflowStreamItem {
4642    pub offset: u64,
4643    pub payload: Option<Value>,
4644    pub payload_envelope: Option<Value>,
4645    pub payload_reference: Option<String>,
4646    pub payload_codec: Option<String>,
4647    pub idempotency_key: Option<String>,
4648    pub item_type: Option<String>,
4649    pub content_type: Option<String>,
4650    pub origin: Option<String>,
4651    pub origin_reference: Option<String>,
4652    pub emitted_at: Option<String>,
4653    pub raw: Value,
4654}
4655
4656/// One bounded at-least-once subscription page.
4657#[derive(Clone, Debug)]
4658pub struct WorkflowStreamPage {
4659    pub stream: WorkflowStreamDescription,
4660    pub items: Vec<WorkflowStreamItem>,
4661    pub next_offset: u64,
4662    pub terminal: bool,
4663}
4664
4665/// Durable acceptance and deduplication outcome for an append request.
4666#[derive(Clone, Debug)]
4667pub struct WorkflowStreamAppendResult {
4668    pub stream: WorkflowStreamDescription,
4669    pub accepted_offsets: Vec<u64>,
4670    pub accepted: u64,
4671    pub deduped: u64,
4672}
4673
4674#[derive(Deserialize)]
4675struct WorkflowStreamListResponse {
4676    #[serde(default)]
4677    streams: Vec<WorkflowStreamDescription>,
4678}
4679
4680#[derive(Deserialize)]
4681struct WorkflowStreamDescriptionResponse {
4682    stream: WorkflowStreamDescription,
4683}
4684
4685#[derive(Deserialize)]
4686struct WorkflowStreamPageResponse {
4687    stream: WorkflowStreamDescription,
4688    #[serde(default)]
4689    items: Vec<Value>,
4690    next_offset: u64,
4691    terminal: bool,
4692}
4693
4694#[derive(Deserialize)]
4695struct WorkflowStreamAppendResponse {
4696    stream: WorkflowStreamDescription,
4697    #[serde(default)]
4698    accepted_offsets: Vec<u64>,
4699    accepted: u64,
4700    deduped: u64,
4701}
4702
4703impl WorkflowDescription {
4704    pub fn is_completed(&self) -> bool {
4705        matches!(self.status.as_deref(), Some("completed" | "Completed"))
4706    }
4707
4708    pub fn is_terminal(&self) -> bool {
4709        matches!(
4710            self.status.as_deref(),
4711            Some(
4712                "completed"
4713                    | "Completed"
4714                    | "failed"
4715                    | "Failed"
4716                    | "cancelled"
4717                    | "Cancelled"
4718                    | "terminated"
4719                    | "Terminated"
4720                    | "timed_out"
4721                    | "TimedOut",
4722            )
4723        )
4724    }
4725
4726    fn decode_payloads(&mut self) -> Result<()> {
4727        if let Some(envelope) = &self.output_envelope {
4728            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
4729            self.output = Some(value.clone().into_json()?);
4730            self.output_avro_value = Some(value);
4731        }
4732
4733        Ok(())
4734    }
4735
4736    fn raw_value(&self) -> Value {
4737        let mut data = self.raw.clone();
4738        data.insert(
4739            "workflow_id".to_string(),
4740            self.workflow_id
4741                .clone()
4742                .map(Value::String)
4743                .unwrap_or(Value::Null),
4744        );
4745        data.insert(
4746            "run_id".to_string(),
4747            self.run_id
4748                .clone()
4749                .map(Value::String)
4750                .unwrap_or(Value::Null),
4751        );
4752        data.insert(
4753            "workflow_type".to_string(),
4754            self.workflow_type
4755                .clone()
4756                .map(Value::String)
4757                .unwrap_or(Value::Null),
4758        );
4759        data.insert(
4760            "status".to_string(),
4761            self.status
4762                .clone()
4763                .map(Value::String)
4764                .unwrap_or(Value::Null),
4765        );
4766        data.insert(
4767            "closed_reason".to_string(),
4768            self.closed_reason
4769                .clone()
4770                .map(Value::String)
4771                .unwrap_or(Value::Null),
4772        );
4773        if let Some(failure) = &self.failure {
4774            data.insert("failure".to_string(), failure.clone());
4775        }
4776        if let Some(exception) = &self.exception {
4777            data.insert("exception".to_string(), exception.clone());
4778        }
4779        Value::Object(data.into_iter().collect())
4780    }
4781}
4782
4783fn workflow_terminal_outcome(
4784    description: &WorkflowDescription,
4785    workflow_id: &str,
4786    run_id: Option<&str>,
4787) -> WorkflowTerminalOutcome {
4788    let terminal_kind = description
4789        .closed_reason
4790        .as_deref()
4791        .or(description.status.as_deref())
4792        .unwrap_or("failed")
4793        .to_ascii_lowercase();
4794    let kind = match terminal_kind.as_str() {
4795        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
4796        "terminated" => WorkflowTerminalKind::Terminated,
4797        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
4798        _ => WorkflowTerminalKind::Failed,
4799    };
4800    let default_reason = match kind {
4801        WorkflowTerminalKind::Failed => "workflow_failed",
4802        WorkflowTerminalKind::Cancelled => "cancelled",
4803        WorkflowTerminalKind::Terminated => "terminated",
4804        WorkflowTerminalKind::TimedOut => "timed_out",
4805    };
4806    let failure = description
4807        .failure
4808        .as_ref()
4809        .filter(|value| value.is_object());
4810    let nested_failure = failure
4811        .and_then(|value| value.get("failures"))
4812        .and_then(Value::as_array)
4813        .and_then(|failures| failures.last())
4814        .or_else(|| description.failures.last());
4815    let exception = description
4816        .exception
4817        .clone()
4818        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
4819        .or_else(|| {
4820            nested_failure
4821                .and_then(|value| value.get("exception_payload"))
4822                .cloned()
4823        });
4824    let string_field = |name: &str| {
4825        failure
4826            .and_then(|value| value.get(name))
4827            .and_then(Value::as_str)
4828            .or_else(|| {
4829                nested_failure
4830                    .and_then(|value| value.get(name))
4831                    .and_then(Value::as_str)
4832            })
4833            .map(str::to_string)
4834    };
4835    let exception_field = |name: &str| {
4836        exception
4837            .as_ref()
4838            .and_then(|value| value.get(name))
4839            .and_then(Value::as_str)
4840            .map(str::to_string)
4841    };
4842    let message = description
4843        .error
4844        .clone()
4845        .or_else(|| string_field("message"))
4846        .or_else(|| exception_field("message"));
4847    let reason = description
4848        .raw
4849        .get("reason")
4850        .and_then(Value::as_str)
4851        .map(str::to_string)
4852        .or_else(|| {
4853            failure
4854                .and_then(|value| value.get("reason"))
4855                .and_then(Value::as_str)
4856                .map(str::to_string)
4857        })
4858        .or_else(|| description.closed_reason.clone())
4859        .unwrap_or_else(|| default_reason.to_string());
4860    let failure_id = string_field("failure_id").or_else(|| {
4861        nested_failure
4862            .and_then(|value| value.get("id"))
4863            .and_then(Value::as_str)
4864            .map(str::to_string)
4865    });
4866
4867    WorkflowTerminalOutcome {
4868        kind,
4869        workflow_id: description
4870            .workflow_id
4871            .clone()
4872            .unwrap_or_else(|| workflow_id.to_string()),
4873        run_id: description
4874            .run_id
4875            .clone()
4876            .or_else(|| run_id.map(str::to_string)),
4877        reason,
4878        failure_category: string_field("failure_category")
4879            .or_else(|| Some(default_reason.to_string())),
4880        failure_id,
4881        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
4882        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
4883        non_retryable: failure
4884            .and_then(|value| value.get("non_retryable"))
4885            .and_then(Value::as_bool)
4886            .or_else(|| {
4887                nested_failure
4888                    .and_then(|value| value.get("non_retryable"))
4889                    .and_then(Value::as_bool)
4890            }),
4891        message,
4892        exception,
4893        raw: description.raw_value(),
4894    }
4895}
4896
4897#[derive(Clone, Debug, Deserialize)]
4898pub struct RegisterWorkerResponse {
4899    pub worker_id: String,
4900    pub registered: bool,
4901    #[serde(default)]
4902    pub heartbeat_interval_seconds: Option<u64>,
4903    #[serde(default)]
4904    pub protocol_version: Option<String>,
4905    #[serde(default)]
4906    pub server_capabilities: Option<Value>,
4907}
4908
4909/// Result of gracefully removing a worker-plane registration.
4910#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
4911pub struct WorkerDeregistrationEnvelope {
4912    pub worker_id: String,
4913    pub outcome: String,
4914    pub recovered_workflow_task_count: u64,
4915}
4916
4917#[derive(Clone, Debug, Deserialize)]
4918pub struct PollWorkflowTaskResponse {
4919    #[serde(default)]
4920    pub task: Option<WorkflowTask>,
4921    #[serde(default)]
4922    pub poll_status: Option<String>,
4923    #[serde(default)]
4924    pub reason: Option<String>,
4925    #[serde(default)]
4926    pub protocol_version: Option<String>,
4927    #[serde(default)]
4928    pub server_capabilities: Option<Value>,
4929}
4930
4931impl PollWorkflowTaskResponse {
4932    /// Classify this response without parsing server display text.
4933    pub fn outcome(&self) -> WorkerPollOutcome {
4934        worker_poll_outcome(
4935            self.task.is_some(),
4936            self.poll_status.as_deref(),
4937            self.reason.as_deref(),
4938        )
4939    }
4940}
4941
4942fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
4943    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
4944        return false;
4945    };
4946    let supported = capabilities
4947        .get("workflow_memo_updates")
4948        .and_then(Value::as_object)
4949        .and_then(|memo| memo.get("supported"))
4950        .and_then(Value::as_bool)
4951        == Some(true);
4952    let command_advertised = capabilities
4953        .get("supported_workflow_task_commands")
4954        .and_then(Value::as_array)
4955        .is_some_and(|commands| {
4956            commands
4957                .iter()
4958                .any(|command| command.as_str() == Some("upsert_memo"))
4959        });
4960    supported && command_advertised
4961}
4962
4963fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
4964    commands
4965        .iter()
4966        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
4967}
4968
4969#[derive(Clone, Debug, Deserialize)]
4970pub struct PollActivityTaskResponse {
4971    #[serde(default)]
4972    pub task: Option<ActivityTask>,
4973    #[serde(default)]
4974    pub poll_status: Option<String>,
4975    #[serde(default)]
4976    pub reason: Option<String>,
4977}
4978
4979impl PollActivityTaskResponse {
4980    /// Classify this response without parsing server display text.
4981    pub fn outcome(&self) -> WorkerPollOutcome {
4982        worker_poll_outcome(
4983            self.task.is_some(),
4984            self.poll_status.as_deref(),
4985            self.reason.as_deref(),
4986        )
4987    }
4988}
4989
4990#[derive(Clone, Debug, Deserialize)]
4991pub struct PollQueryTaskResponse {
4992    #[serde(default)]
4993    pub task: Option<QueryTask>,
4994    #[serde(default)]
4995    pub poll_status: Option<String>,
4996    #[serde(default)]
4997    pub reason: Option<String>,
4998}
4999
5000impl PollQueryTaskResponse {
5001    /// Classify this response without parsing server display text.
5002    pub fn outcome(&self) -> WorkerPollOutcome {
5003        worker_poll_outcome(
5004            self.task.is_some(),
5005            self.poll_status.as_deref(),
5006            self.reason.as_deref(),
5007        )
5008    }
5009}
5010
5011/// Stable classification for worker poll responses.
5012#[derive(Clone, Debug, PartialEq, Eq)]
5013pub enum WorkerPollOutcome {
5014    /// A task was leased and is available on the response.
5015    Task,
5016    /// No task was leased, but the worker should continue polling.
5017    Idle {
5018        poll_status: Option<String>,
5019        reason: Option<String>,
5020    },
5021    /// The server asked this worker to stop claiming new work.
5022    Stop {
5023        poll_status: Option<String>,
5024        reason: Option<String>,
5025    },
5026}
5027
5028impl WorkerPollOutcome {
5029    pub fn should_stop(&self) -> bool {
5030        matches!(self, Self::Stop { .. })
5031    }
5032}
5033
5034fn worker_poll_outcome(
5035    has_task: bool,
5036    poll_status: Option<&str>,
5037    reason: Option<&str>,
5038) -> WorkerPollOutcome {
5039    if worker_poll_is_stop(poll_status, reason) {
5040        return WorkerPollOutcome::Stop {
5041            poll_status: poll_status.map(str::to_string),
5042            reason: reason.map(str::to_string),
5043        };
5044    }
5045
5046    if has_task {
5047        WorkerPollOutcome::Task
5048    } else {
5049        WorkerPollOutcome::Idle {
5050            poll_status: poll_status.map(str::to_string),
5051            reason: reason.map(str::to_string),
5052        }
5053    }
5054}
5055
5056/// An ephemeral server-routed query task.
5057#[derive(Clone, Debug, Deserialize)]
5058pub struct QueryTask {
5059    pub query_task_id: String,
5060    #[serde(default = "default_workflow_task_attempt")]
5061    pub query_task_attempt: u64,
5062    #[serde(default)]
5063    pub lease_owner: Option<String>,
5064    #[serde(default)]
5065    pub workflow_id: Option<String>,
5066    #[serde(default)]
5067    pub run_id: Option<String>,
5068    pub workflow_type: String,
5069    pub query_name: String,
5070    #[serde(
5071        default = "missing_task_payload_codec",
5072        deserialize_with = "deserialize_task_payload_codec"
5073    )]
5074    pub payload_codec: String,
5075    #[serde(default)]
5076    pub workflow_arguments: Option<Value>,
5077    #[serde(default)]
5078    pub query_arguments: Option<Value>,
5079    #[serde(default)]
5080    pub history_events: Vec<HistoryEvent>,
5081    #[serde(default)]
5082    pub history_export: Option<Value>,
5083    #[serde(default)]
5084    pub run_status: Option<String>,
5085}
5086
5087#[derive(Clone, Debug, Deserialize)]
5088pub struct WorkflowTask {
5089    pub task_id: String,
5090    #[serde(default)]
5091    pub workflow_command_id: Option<String>,
5092    #[serde(default)]
5093    pub workflow_id: Option<String>,
5094    #[serde(default)]
5095    pub run_id: Option<String>,
5096    pub workflow_type: String,
5097    #[serde(default)]
5098    pub cancel_requested: bool,
5099    #[serde(
5100        default = "missing_task_payload_codec",
5101        deserialize_with = "deserialize_task_payload_codec"
5102    )]
5103    pub payload_codec: String,
5104    #[serde(default)]
5105    pub arguments: Option<Value>,
5106    #[serde(default)]
5107    pub history_events: Vec<HistoryEvent>,
5108    #[serde(default)]
5109    pub total_history_events: Option<u64>,
5110    #[serde(default)]
5111    pub history_size_bytes: Option<u64>,
5112    #[serde(default)]
5113    pub continue_as_new_recommended: Option<bool>,
5114    #[serde(default)]
5115    pub history_budget_pressure: Option<String>,
5116    #[serde(default)]
5117    pub next_history_page_token: Option<String>,
5118    #[serde(default = "default_workflow_task_attempt")]
5119    pub workflow_task_attempt: u64,
5120    #[serde(default)]
5121    pub workflow_signal_id: Option<String>,
5122    #[serde(default)]
5123    pub signal_name: Option<String>,
5124    #[serde(default)]
5125    pub signal_arguments: Option<Value>,
5126    #[serde(default)]
5127    pub workflow_update_id: Option<String>,
5128    #[serde(default)]
5129    pub update_name: Option<String>,
5130    #[serde(default)]
5131    pub lease_owner: Option<String>,
5132}
5133
5134impl WorkflowTask {
5135    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5136        self.history_events.extend(page.history_events);
5137
5138        if page.total_history_events.is_some() {
5139            self.total_history_events = page.total_history_events;
5140        }
5141
5142        self.next_history_page_token = page
5143            .next_history_page_token
5144            .filter(|token| !token.is_empty());
5145    }
5146}
5147
5148#[derive(Clone, Debug, Deserialize)]
5149struct WorkflowTaskHistoryPage {
5150    #[serde(default)]
5151    history_events: Vec<HistoryEvent>,
5152    #[serde(default)]
5153    total_history_events: Option<u64>,
5154    #[serde(default)]
5155    next_history_page_token: Option<String>,
5156}
5157
5158#[derive(Clone, Debug, Deserialize)]
5159pub struct ActivityTask {
5160    pub task_id: String,
5161    #[serde(default)]
5162    pub activity_attempt_id: Option<String>,
5163    #[serde(default)]
5164    pub attempt_id: Option<String>,
5165    pub activity_type: String,
5166    #[serde(
5167        default = "missing_task_payload_codec",
5168        deserialize_with = "deserialize_task_payload_codec"
5169    )]
5170    pub payload_codec: String,
5171    #[serde(default)]
5172    pub arguments: Option<Value>,
5173    #[serde(default = "default_attempt_number")]
5174    pub attempt_number: u64,
5175    #[serde(default)]
5176    pub lease_owner: Option<String>,
5177}
5178
5179#[derive(Clone, Debug, Deserialize)]
5180pub struct HistoryEvent {
5181    #[serde(alias = "type")]
5182    pub event_type: String,
5183    #[serde(default)]
5184    pub payload: Value,
5185    #[serde(flatten)]
5186    pub raw: HashMap<String, Value>,
5187}
5188
5189/// One decoded signal in the committed workflow-history snapshot.
5190#[derive(Clone, Debug, PartialEq)]
5191pub struct QuerySignal {
5192    pub id: Option<String>,
5193    pub name: String,
5194    pub arguments: Vec<Value>,
5195    avro_arguments: Vec<AvroValue>,
5196    pub workflow_sequence: Option<u64>,
5197}
5198
5199impl QuerySignal {
5200    /// Lossless fixed Avro Value arguments for this committed signal.
5201    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5202        &self.avro_arguments
5203    }
5204}
5205
5206/// Immutable state supplied to a registered query handler.
5207///
5208/// This context intentionally exposes no activity, signal-wait, or command
5209/// APIs. Query handlers inspect committed history and return a value; query
5210/// completion does not append an event or advance deterministic execution.
5211#[derive(Clone, Debug)]
5212pub struct QueryContext {
5213    pub workflow_id: Option<String>,
5214    pub run_id: Option<String>,
5215    pub workflow_type: String,
5216    pub run_status: Option<String>,
5217    workflow_input: Value,
5218    workflow_input_avro_value: AvroValue,
5219    history_events: Arc<Vec<HistoryEvent>>,
5220    signal_events: Arc<Vec<QuerySignal>>,
5221}
5222
5223impl QueryContext {
5224    /// The normalized argument list used to start the workflow.
5225    pub fn workflow_input(&self) -> &Value {
5226        &self.workflow_input
5227    }
5228
5229    /// The lossless fixed Avro Value argument list used to start the workflow.
5230    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5231        &self.workflow_input_avro_value
5232    }
5233
5234    /// The immutable committed history used for this query snapshot.
5235    pub fn history_events(&self) -> &[HistoryEvent] {
5236        self.history_events.as_slice()
5237    }
5238
5239    /// All decoded signals in committed workflow order.
5240    pub fn signal_events(&self) -> &[QuerySignal] {
5241        self.signal_events.as_slice()
5242    }
5243
5244    /// Decoded argument lists for each committed signal with `signal_name`.
5245    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5246        self.signal_events
5247            .iter()
5248            .filter(|signal| signal.name == signal_name)
5249            .map(|signal| signal.arguments.clone())
5250            .collect()
5251    }
5252
5253    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5254    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5255        self.signal_events
5256            .iter()
5257            .filter(|signal| signal.name == signal_name)
5258            .map(|signal| signal.avro_arguments.clone())
5259            .collect()
5260    }
5261}
5262
5263#[derive(Clone, Debug, Deserialize)]
5264pub struct ActivityHeartbeatResponse {
5265    #[serde(default)]
5266    pub cancel_requested: bool,
5267    #[serde(default)]
5268    pub heartbeat_recorded: bool,
5269    #[serde(default)]
5270    pub can_continue: Option<bool>,
5271    #[serde(default)]
5272    pub reason: Option<String>,
5273    #[serde(default)]
5274    pub run_closed_reason: Option<String>,
5275    #[serde(default)]
5276    pub run_closed_at: Option<String>,
5277    #[serde(default)]
5278    pub lease_expires_at: Option<String>,
5279    #[serde(default)]
5280    pub last_heartbeat_at: Option<String>,
5281}
5282
5283impl ActivityHeartbeatResponse {
5284    /// Whether the activity should stop instead of attempting completion.
5285    pub fn should_stop(&self) -> bool {
5286        self.cancel_requested || self.can_continue == Some(false)
5287    }
5288}
5289
5290fn missing_task_payload_codec() -> String {
5291    MISSING_TASK_PAYLOAD_CODEC.to_string()
5292}
5293
5294fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5295where
5296    D: Deserializer<'de>,
5297{
5298    Ok(match Value::deserialize(deserializer)? {
5299        Value::String(codec) => codec,
5300        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5301        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5302    })
5303}
5304
5305fn default_workflow_task_attempt() -> u64 {
5306    1
5307}
5308
5309fn default_attempt_number() -> u64 {
5310    1
5311}
5312
5313type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5314type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5315type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5316type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5317type ReplayedWorkflowHandler =
5318    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5319type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5320type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5321type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5322type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5323type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5324type ReplayedQueryHandler = Arc<
5325    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5326        + Send
5327        + Sync,
5328>;
5329type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5330
5331struct ReplayedWorkflowInvocation {
5332    future: WorkflowFuture,
5333    snapshot: WorkflowStateSnapshot,
5334}
5335
5336#[derive(Clone)]
5337struct RegisteredWorkflow {
5338    execute: WorkflowHandler,
5339    replay: Option<ReplayedWorkflowHandler>,
5340    state_type: Option<TypeId>,
5341}
5342
5343#[derive(Clone)]
5344enum RegisteredQuery {
5345    Snapshot(QueryHandler),
5346    Replayed {
5347        state_type: TypeId,
5348        handler: ReplayedQueryHandler,
5349    },
5350}
5351
5352#[derive(Clone, Debug)]
5353pub struct WorkerHeartbeatObservation {
5354    pub worker_id: String,
5355    pub task_queue: String,
5356    pub acknowledged_at_unix_millis: u64,
5357    pub acknowledgement: Value,
5358}
5359
5360/// Bounded retry policy for worker poll acquisition and worker heartbeats.
5361///
5362/// Expected empty long polls are normal successful responses. Transport
5363/// failures, HTTP 408/429 responses, and server errors are retried with capped
5364/// exponential backoff. Authentication, protocol, codec, and handler failures
5365/// are never retried by the worker.
5366#[derive(Clone, Copy, Debug)]
5367pub struct WorkerRetryPolicy {
5368    /// Number of retries after the initial request fails.
5369    pub max_retries: usize,
5370    /// Delay before the first retry.
5371    pub initial_backoff: Duration,
5372    /// Maximum delay between retries.
5373    pub max_backoff: Duration,
5374}
5375
5376impl Default for WorkerRetryPolicy {
5377    fn default() -> Self {
5378        Self {
5379            max_retries: 5,
5380            initial_backoff: Duration::from_millis(100),
5381            max_backoff: Duration::from_secs(5),
5382        }
5383    }
5384}
5385
5386#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5387enum ManagedPollOutcome {
5388    Idle,
5389    Handled,
5390    Stop,
5391}
5392
5393#[derive(Clone)]
5394pub struct Worker {
5395    client: Client,
5396    worker_id: String,
5397    task_queue: String,
5398    workflows: HashMap<String, RegisteredWorkflow>,
5399    activities: HashMap<String, ActivityHandler>,
5400    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5401    updates: HashMap<String, HashMap<String, UpdateHandler>>,
5402    max_concurrent_workflow_tasks: usize,
5403    max_concurrent_activity_tasks: usize,
5404    poll_timeout: Duration,
5405    heartbeat_interval: Duration,
5406    retry_policy: WorkerRetryPolicy,
5407    heartbeat_observer: Option<WorkerHeartbeatObserver>,
5408}
5409
5410impl Worker {
5411    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5412        Self {
5413            client,
5414            worker_id: default_worker_id(),
5415            task_queue: task_queue.into(),
5416            workflows: HashMap::new(),
5417            activities: HashMap::new(),
5418            queries: HashMap::new(),
5419            updates: HashMap::new(),
5420            max_concurrent_workflow_tasks: 10,
5421            max_concurrent_activity_tasks: 10,
5422            poll_timeout: Duration::from_secs(30),
5423            heartbeat_interval: Duration::from_secs(60),
5424            retry_policy: WorkerRetryPolicy::default(),
5425            heartbeat_observer: None,
5426        }
5427    }
5428
5429    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5430        self.worker_id = worker_id.into();
5431        self
5432    }
5433
5434    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5435        self.poll_timeout = timeout;
5436        self
5437    }
5438
5439    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5440        self.heartbeat_interval = interval;
5441        self
5442    }
5443
5444    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
5445    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5446        self.retry_policy = policy;
5447        self
5448    }
5449
5450    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5451    where
5452        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5453    {
5454        self.heartbeat_observer = Some(Arc::new(observer));
5455        self
5456    }
5457
5458    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5459        self.max_concurrent_workflow_tasks = count.max(1);
5460        self
5461    }
5462
5463    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5464        self.max_concurrent_activity_tasks = count.max(1);
5465        self
5466    }
5467
5468    /// Register a workflow handler.
5469    ///
5470    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
5471    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
5472    /// while acquiring or decoding a worker task remain worker-operation
5473    /// failures and do not get converted into workflow outcomes.
5474    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5475    where
5476        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5477        Fut: Future<Output = Result<Value>> + Send + 'static,
5478    {
5479        let handler = Arc::new(handler);
5480        self.workflows.insert(
5481            workflow_type.into(),
5482            RegisteredWorkflow {
5483                execute: Arc::new(move |ctx, input| {
5484                    let handler = Arc::clone(&handler);
5485                    Box::pin(async move {
5486                        let result = handler(ctx, input.into_json()?).await?;
5487                        AvroValue::from_serialize(&result)
5488                    })
5489                }),
5490                replay: None,
5491                state_type: None,
5492            },
5493        );
5494    }
5495
5496    /// Register a workflow with one Serde request value and a Serde result.
5497    ///
5498    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
5499    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
5500    /// workflow-specific schema. A task must contain zero arguments for a unit
5501    /// request or exactly one argument for every other request type.
5502    ///
5503    /// See the runnable
5504    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
5505    /// for typed workflow and activity contracts with retry and timeout policy.
5506    pub fn register_typed_workflow<I, O, F, Fut>(
5507        &mut self,
5508        workflow_type: impl Into<String>,
5509        handler: F,
5510    ) where
5511        I: DeserializeOwned + Send + 'static,
5512        O: Serialize + Send + 'static,
5513        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
5514        Fut: Future<Output = Result<O>> + Send + 'static,
5515    {
5516        let workflow_type = workflow_type.into();
5517        let handler_name = workflow_type.clone();
5518        let handler = Arc::new(handler);
5519        self.workflows.insert(
5520            workflow_type,
5521            RegisteredWorkflow {
5522                execute: Arc::new(move |ctx, input| {
5523                    let handler = Arc::clone(&handler);
5524                    let handler_name = handler_name.clone();
5525                    Box::pin(async move {
5526                        let input =
5527                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5528                        let result = handler(ctx, input).await?;
5529                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5530                    })
5531                }),
5532                replay: None,
5533                state_type: None,
5534            },
5535        );
5536    }
5537
5538    /// Register a workflow on the lossless fixed Avro Value surface.
5539    pub fn register_workflow_avro_value<F, Fut>(
5540        &mut self,
5541        workflow_type: impl Into<String>,
5542        handler: F,
5543    ) where
5544        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
5545        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5546    {
5547        self.workflows.insert(
5548            workflow_type.into(),
5549            RegisteredWorkflow {
5550                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
5551                replay: None,
5552                state_type: None,
5553            },
5554        );
5555    }
5556
5557    /// Register a workflow whose typed instance state can be reconstructed for queries.
5558    ///
5559    /// `state_factory` creates a fresh instance for every normal workflow task and
5560    /// query replay. The workflow handler is the single source of truth for state
5561    /// transitions: it updates [`WorkflowInstance`] after activities and signals
5562    /// resolve. Query replay runs this same handler over committed history and
5563    /// discards any commands it would emit.
5564    pub fn register_replayed_workflow<S, Factory, F, Fut>(
5565        &mut self,
5566        workflow_type: impl Into<String>,
5567        state_factory: Factory,
5568        handler: F,
5569    ) where
5570        S: Clone + Send + Sync + 'static,
5571        Factory: Fn() -> S + Send + Sync + 'static,
5572        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5573        Fut: Future<Output = Result<Value>> + Send + 'static,
5574    {
5575        let state_factory = Arc::new(state_factory);
5576        let handler = Arc::new(handler);
5577
5578        let execute_factory = Arc::clone(&state_factory);
5579        let execute_handler = Arc::clone(&handler);
5580        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5581            let state = WorkflowInstance::new(execute_factory());
5582            let handler = Arc::clone(&execute_handler);
5583            Box::pin(async move {
5584                let result = handler(ctx, input.into_json()?, state).await?;
5585                AvroValue::from_serialize(&result)
5586            }) as WorkflowFuture
5587        });
5588
5589        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5590            let state = WorkflowInstance::new(state_factory());
5591            let snapshot_state = state.clone();
5592            let snapshot: WorkflowStateSnapshot =
5593                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5594            let replay_handler = Arc::clone(&handler);
5595            let future = async move {
5596                let result = replay_handler(ctx, input.into_json()?, state).await?;
5597                AvroValue::from_serialize(&result)
5598            };
5599            ReplayedWorkflowInvocation {
5600                future: Box::pin(future),
5601                snapshot,
5602            }
5603        });
5604
5605        self.workflows.insert(
5606            workflow_type.into(),
5607            RegisteredWorkflow {
5608                execute,
5609                replay: Some(replay),
5610                state_type: Some(TypeId::of::<S>()),
5611            },
5612        );
5613    }
5614
5615    /// Register a replayable workflow with one Serde request value and result.
5616    ///
5617    /// Normal task execution and instance-state query replay both decode and
5618    /// encode through the fixed Avro Value codec. The state factory and handler
5619    /// otherwise follow [`Worker::register_replayed_workflow`].
5620    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
5621        &mut self,
5622        workflow_type: impl Into<String>,
5623        state_factory: Factory,
5624        handler: F,
5625    ) where
5626        I: DeserializeOwned + Send + 'static,
5627        O: Serialize + Send + 'static,
5628        S: Clone + Send + Sync + 'static,
5629        Factory: Fn() -> S + Send + Sync + 'static,
5630        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5631        Fut: Future<Output = Result<O>> + Send + 'static,
5632    {
5633        let workflow_type = workflow_type.into();
5634        let state_factory = Arc::new(state_factory);
5635        let handler = Arc::new(handler);
5636
5637        let execute_name = workflow_type.clone();
5638        let execute_factory = Arc::clone(&state_factory);
5639        let execute_handler = Arc::clone(&handler);
5640        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5641            let state = WorkflowInstance::new(execute_factory());
5642            let handler = Arc::clone(&execute_handler);
5643            let handler_name = execute_name.clone();
5644            Box::pin(async move {
5645                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5646                let result = handler(ctx, input, state).await?;
5647                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5648            }) as WorkflowFuture
5649        });
5650
5651        let replay_name = workflow_type.clone();
5652        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5653            let state = WorkflowInstance::new(state_factory());
5654            let snapshot_state = state.clone();
5655            let snapshot: WorkflowStateSnapshot =
5656                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5657            let handler = Arc::clone(&handler);
5658            let handler_name = replay_name.clone();
5659            let future = async move {
5660                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
5661                let result = handler(ctx, input, state).await?;
5662                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
5663            };
5664            ReplayedWorkflowInvocation {
5665                future: Box::pin(future),
5666                snapshot,
5667            }
5668        });
5669
5670        self.workflows.insert(
5671            workflow_type,
5672            RegisteredWorkflow {
5673                execute,
5674                replay: Some(replay),
5675                state_type: Some(TypeId::of::<S>()),
5676            },
5677        );
5678    }
5679
5680    /// Register a replayable workflow on the lossless fixed Avro Value surface.
5681    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
5682        &mut self,
5683        workflow_type: impl Into<String>,
5684        state_factory: Factory,
5685        handler: F,
5686    ) where
5687        S: Clone + Send + Sync + 'static,
5688        Factory: Fn() -> S + Send + Sync + 'static,
5689        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
5690        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5691    {
5692        let state_factory = Arc::new(state_factory);
5693        let handler = Arc::new(handler);
5694
5695        let execute_factory = Arc::clone(&state_factory);
5696        let execute_handler = Arc::clone(&handler);
5697        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5698            let state = WorkflowInstance::new(execute_factory());
5699            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
5700        });
5701
5702        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
5703            let state = WorkflowInstance::new(state_factory());
5704            let snapshot_state = state.clone();
5705            let snapshot: WorkflowStateSnapshot =
5706                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
5707            ReplayedWorkflowInvocation {
5708                future: Box::pin(handler(ctx, input, state)),
5709                snapshot,
5710            }
5711        });
5712
5713        self.workflows.insert(
5714            workflow_type.into(),
5715            RegisteredWorkflow {
5716                execute,
5717                replay: Some(replay),
5718                state_type: Some(TypeId::of::<S>()),
5719            },
5720        );
5721    }
5722
5723    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
5724    where
5725        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
5726        Fut: Future<Output = Result<Value>> + Send + 'static,
5727    {
5728        let handler = Arc::new(handler);
5729        self.activities.insert(
5730            activity_type.into(),
5731            Arc::new(move |ctx, args| {
5732                let handler = Arc::clone(&handler);
5733                Box::pin(async move {
5734                    let result = handler(ctx, args.into_json()?).await?;
5735                    AvroValue::from_serialize(&result)
5736                })
5737            }),
5738        );
5739    }
5740
5741    /// Register an activity with one Serde request value and a Serde result.
5742    ///
5743    /// Inputs and results use the platform's fixed Avro Value schema. Shape
5744    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
5745    /// with the activity name and Rust type.
5746    pub fn register_typed_activity<I, O, F, Fut>(
5747        &mut self,
5748        activity_type: impl Into<String>,
5749        handler: F,
5750    ) where
5751        I: DeserializeOwned + Send + 'static,
5752        O: Serialize + Send + 'static,
5753        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
5754        Fut: Future<Output = Result<O>> + Send + 'static,
5755    {
5756        let activity_type = activity_type.into();
5757        let handler_name = activity_type.clone();
5758        let handler = Arc::new(handler);
5759        self.activities.insert(
5760            activity_type,
5761            Arc::new(move |ctx, input| {
5762                let handler = Arc::clone(&handler);
5763                let handler_name = handler_name.clone();
5764                Box::pin(async move {
5765                    let input =
5766                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
5767                    let result = handler(ctx, input).await?;
5768                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
5769                })
5770            }),
5771        );
5772    }
5773
5774    /// Register an activity on the lossless fixed Avro Value surface.
5775    pub fn register_activity_avro_value<F, Fut>(
5776        &mut self,
5777        activity_type: impl Into<String>,
5778        handler: F,
5779    ) where
5780        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
5781        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5782    {
5783        self.activities.insert(
5784            activity_type.into(),
5785            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
5786        );
5787    }
5788
5789    /// Register a named, read-only query handler for a workflow type.
5790    ///
5791    /// The workflow type must also be registered with [`Worker::register_workflow`]
5792    /// before the worker runs. The handler receives only an immutable committed
5793    /// state snapshot and normalized query arguments.
5794    pub fn register_query<F, Fut>(
5795        &mut self,
5796        workflow_type: impl Into<String>,
5797        query_name: impl Into<String>,
5798        handler: F,
5799    ) where
5800        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
5801        Fut: Future<Output = Result<Value>> + Send + 'static,
5802    {
5803        let handler = Arc::new(handler);
5804        self.queries
5805            .entry(workflow_type.into())
5806            .or_default()
5807            .insert(
5808                query_name.into(),
5809                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
5810                    let handler = Arc::clone(&handler);
5811                    Box::pin(async move {
5812                        let result = handler(ctx, args.into_json()?).await?;
5813                        AvroValue::from_serialize(&result)
5814                    })
5815                })),
5816            );
5817    }
5818
5819    /// Register a query handler on the lossless fixed Avro Value surface.
5820    pub fn register_query_avro_value<F, Fut>(
5821        &mut self,
5822        workflow_type: impl Into<String>,
5823        query_name: impl Into<String>,
5824        handler: F,
5825    ) where
5826        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
5827        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5828    {
5829        self.queries
5830            .entry(workflow_type.into())
5831            .or_default()
5832            .insert(
5833                query_name.into(),
5834                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
5835            );
5836    }
5837
5838    /// Register a named query against deterministically replayed instance state.
5839    ///
5840    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
5841    /// same state type `S`. The handler receives an immutable, detached state
5842    /// clone, so successful and failed queries cannot affect workflow execution
5843    /// or the state reconstructed by a later query.
5844    pub fn register_replayed_query<S, F, Fut>(
5845        &mut self,
5846        workflow_type: impl Into<String>,
5847        query_name: impl Into<String>,
5848        handler: F,
5849    ) where
5850        S: Clone + Send + Sync + 'static,
5851        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
5852        Fut: Future<Output = Result<Value>> + Send + 'static,
5853    {
5854        let handler = Arc::new(handler);
5855        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
5856            let state = state.downcast::<S>().map_err(|_| {
5857                "registered query state type does not match the replayed workflow state".to_string()
5858            })?;
5859            let handler = Arc::clone(&handler);
5860            Ok(Box::pin(async move {
5861                let result = handler(ctx, state, args.into_json()?).await?;
5862                AvroValue::from_serialize(&result)
5863            }))
5864        });
5865
5866        self.queries
5867            .entry(workflow_type.into())
5868            .or_default()
5869            .insert(
5870                query_name.into(),
5871                RegisteredQuery::Replayed {
5872                    state_type: TypeId::of::<S>(),
5873                    handler: erased_handler,
5874                },
5875            );
5876    }
5877
5878    /// Register a replayed-state query on the lossless fixed Avro Value surface.
5879    pub fn register_replayed_query_avro_value<S, F, Fut>(
5880        &mut self,
5881        workflow_type: impl Into<String>,
5882        query_name: impl Into<String>,
5883        handler: F,
5884    ) where
5885        S: Clone + Send + Sync + 'static,
5886        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
5887        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5888    {
5889        let handler = Arc::new(handler);
5890        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
5891            let state = state.downcast::<S>().map_err(|_| {
5892                "registered query state type does not match the replayed workflow state".to_string()
5893            })?;
5894            Ok(Box::pin(handler(ctx, state, args)))
5895        });
5896
5897        self.queries
5898            .entry(workflow_type.into())
5899            .or_default()
5900            .insert(
5901                query_name.into(),
5902                RegisteredQuery::Replayed {
5903                    state_type: TypeId::of::<S>(),
5904                    handler: erased_handler,
5905                },
5906            );
5907    }
5908
5909    /// Register a synchronous workflow update handler.
5910    pub fn register_update<F, Fut>(
5911        &mut self,
5912        workflow_type: impl Into<String>,
5913        update_name: impl Into<String>,
5914        handler: F,
5915    ) where
5916        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
5917        Fut: Future<Output = Result<Value>> + Send + 'static,
5918    {
5919        let handler = Arc::new(handler);
5920        self.updates
5921            .entry(workflow_type.into())
5922            .or_default()
5923            .insert(
5924                update_name.into(),
5925                Arc::new(move |ctx, args| {
5926                    let handler = Arc::clone(&handler);
5927                    Box::pin(async move {
5928                        let result = handler(ctx, args.into_json()?).await?;
5929                        AvroValue::from_serialize(&result)
5930                    })
5931                }),
5932            );
5933    }
5934
5935    /// Register an update handler on the lossless fixed Avro Value surface.
5936    pub fn register_update_avro_value<F, Fut>(
5937        &mut self,
5938        workflow_type: impl Into<String>,
5939        update_name: impl Into<String>,
5940        handler: F,
5941    ) where
5942        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
5943        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
5944    {
5945        self.updates
5946            .entry(workflow_type.into())
5947            .or_default()
5948            .insert(
5949                update_name.into(),
5950                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
5951            );
5952    }
5953
5954    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
5955        let mut command_contracts = serde_json::Map::new();
5956        for workflow_type in self.workflows.keys() {
5957            let mut queries = self
5958                .queries
5959                .get(workflow_type)
5960                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
5961                .unwrap_or_default();
5962            queries.sort();
5963            let mut updates = self
5964                .updates
5965                .get(workflow_type)
5966                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
5967                .unwrap_or_default();
5968            updates.sort();
5969            if !queries.is_empty() || !updates.is_empty() {
5970                command_contracts.insert(
5971                    workflow_type.clone(),
5972                    json!({
5973                        "queries": queries,
5974                        "updates": updates,
5975                        "update_validators": [],
5976                    }),
5977                );
5978            }
5979        }
5980
5981        self.client
5982            .register_worker_with_command_contracts(
5983                &self.worker_id,
5984                &self.task_queue,
5985                self.workflows.keys().cloned().collect(),
5986                self.activities.keys().cloned().collect(),
5987                self.max_concurrent_workflow_tasks,
5988                self.max_concurrent_activity_tasks,
5989                [
5990                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
5991                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
5992                ]
5993                .into_iter()
5994                .flatten()
5995                .collect(),
5996                Value::Object(command_contracts),
5997            )
5998            .await
5999    }
6000
6001    /// Run until shutdown or a terminal worker error occurs.
6002    ///
6003    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6004    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6005    /// authentication, protocol, and other non-retryable failures are returned.
6006    pub async fn run(&self) -> Result<()> {
6007        self.run_until(std::future::pending::<()>()).await
6008    }
6009
6010    /// Run until `shutdown` resolves or a terminal worker error occurs.
6011    ///
6012    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6013    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6014    where
6015        F: Future<Output = ()>,
6016    {
6017        let registration = self.register().await?;
6018        if !registration.registered {
6019            return Err(Error::WorkerLoop(format!(
6020                "worker registration for {:?} was not accepted",
6021                self.worker_id
6022            )));
6023        }
6024        let registered_worker_id = registration.worker_id.clone();
6025        let primary = self.run_registered_until(shutdown, registration).await;
6026        let deregistration = self
6027            .client
6028            .deregister_worker_registration(&registered_worker_id)
6029            .await;
6030
6031        match (primary, deregistration) {
6032            (Ok(()), Ok(_)) => Ok(()),
6033            (Ok(()), Err(deregistration)) => Err(deregistration),
6034            (Err(primary), Ok(_)) => Err(primary),
6035            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6036                primary: Box::new(primary),
6037                deregistration: Box::new(deregistration),
6038            }),
6039        }
6040    }
6041
6042    async fn run_registered_until<F>(
6043        &self,
6044        shutdown: F,
6045        registration: RegisterWorkerResponse,
6046    ) -> Result<()>
6047    where
6048        F: Future<Output = ()>,
6049    {
6050        let heartbeat_interval = Duration::from_secs(
6051            registration
6052                .heartbeat_interval_seconds
6053                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6054        );
6055        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6056        // from the completion of the preceding attempt, including its bounded
6057        // retries. A fixed-epoch interval can leave an already-due tick queued
6058        // while an acknowledgement is slow, producing a catch-up heartbeat as
6059        // soon as that request completes.
6060        let heartbeat = tokio::time::sleep(Duration::ZERO);
6061        tokio::pin!(heartbeat);
6062        tokio::pin!(shutdown);
6063        let stop = Arc::new(AtomicBool::new(false));
6064        // Poll responses may already have leased server-side work by the time
6065        // they become ready, so each poller owns its responses through
6066        // completion or failure instead of racing raw polls in this select.
6067        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6068            let worker = self.clone();
6069            let stop = Arc::clone(&stop);
6070            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6071        });
6072        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6073            let worker = self.clone();
6074            let stop = Arc::clone(&stop);
6075            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6076        });
6077        let mut query_poller = (!self.queries.is_empty()).then(|| {
6078            let worker = self.clone();
6079            let stop = Arc::clone(&stop);
6080            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6081        });
6082
6083        loop {
6084            tokio::select! {
6085                _ = &mut shutdown => {
6086                    stop.store(true, Ordering::SeqCst);
6087                    break;
6088                }
6089                _ = &mut heartbeat => {
6090                    let result = self.retry_worker_operation(|| {
6091                        self.client.heartbeat_worker(
6092                            &self.worker_id,
6093                            self.max_concurrent_workflow_tasks,
6094                            self.max_concurrent_activity_tasks,
6095                        )
6096                    }).await;
6097                    heartbeat
6098                        .as_mut()
6099                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6100                    match result {
6101                        Ok(acknowledgement) => {
6102                            if let Some(observer) = &self.heartbeat_observer {
6103                                observer(&WorkerHeartbeatObservation {
6104                                    worker_id: self.worker_id.clone(),
6105                                    task_queue: self.task_queue.clone(),
6106                                    acknowledged_at_unix_millis: SystemTime::now()
6107                                        .duration_since(UNIX_EPOCH)
6108                                        .unwrap_or_default()
6109                                        .as_millis()
6110                                        .min(u64::MAX as u128)
6111                                        as u64,
6112                                    acknowledgement,
6113                                });
6114                            }
6115                        }
6116                        Err(error) => {
6117                            stop.store(true, Ordering::SeqCst);
6118                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6119                            return Err(error);
6120                        }
6121                    }
6122                }
6123                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6124                    workflow_poller = None;
6125                    let stopped_by_server = stop.load(Ordering::SeqCst);
6126                    stop.store(true, Ordering::SeqCst);
6127                    let poller_result = optional_poller_result("workflow", result);
6128                    let join_result =
6129                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6130                    poller_result?;
6131                    join_result?;
6132                    if stopped_by_server {
6133                        return Ok(());
6134                    }
6135                    return Err(Error::WorkerLoop(
6136                        "workflow poller stopped unexpectedly".to_string(),
6137                    ));
6138                }
6139                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6140                    activity_poller = None;
6141                    let stopped_by_server = stop.load(Ordering::SeqCst);
6142                    stop.store(true, Ordering::SeqCst);
6143                    let poller_result = optional_poller_result("activity", result);
6144                    let join_result =
6145                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6146                    poller_result?;
6147                    join_result?;
6148                    if stopped_by_server {
6149                        return Ok(());
6150                    }
6151                    return Err(Error::WorkerLoop(
6152                        "activity poller stopped unexpectedly".to_string(),
6153                    ));
6154                }
6155                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6156                    query_poller = None;
6157                    let stopped_by_server = stop.load(Ordering::SeqCst);
6158                    stop.store(true, Ordering::SeqCst);
6159                    let poller_result = optional_poller_result("query", result);
6160                    let join_result =
6161                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6162                    poller_result?;
6163                    join_result?;
6164                    if stopped_by_server {
6165                        return Ok(());
6166                    }
6167                    return Err(Error::WorkerLoop(
6168                        "query poller stopped unexpectedly".to_string(),
6169                    ));
6170                }
6171            }
6172        }
6173
6174        join_pollers(
6175            workflow_poller.take(),
6176            activity_poller.take(),
6177            query_poller.take(),
6178        )
6179        .await
6180    }
6181
6182    /// Poll and settle at most one task from each enabled task family.
6183    ///
6184    /// A workflow may reach its server-enforced run deadline while this worker
6185    /// holds a task. When the completion endpoint authoritatively rejects that
6186    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
6187    /// terminal `run_status=failed`, the workflow tick is considered settled:
6188    /// the late command was not recorded and cannot replace the terminal run.
6189    /// Every other completion rejection remains an error. This worker-level
6190    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
6191    /// only bounds how long a client waits for a result.
6192    ///
6193    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
6194    /// the original [`Error::Http`] status and response body.
6195    pub async fn run_once(&self) -> Result<usize> {
6196        let mut handled = 0;
6197        match self.poll_workflow_once().await? {
6198            ManagedPollOutcome::Handled => handled += 1,
6199            ManagedPollOutcome::Stop => return Ok(handled),
6200            ManagedPollOutcome::Idle => {}
6201        }
6202        match self.poll_activity_once().await? {
6203            ManagedPollOutcome::Handled => handled += 1,
6204            ManagedPollOutcome::Stop => return Ok(handled),
6205            ManagedPollOutcome::Idle => {}
6206        }
6207        if !self.queries.is_empty() {
6208            match self.poll_query_once().await? {
6209                ManagedPollOutcome::Handled => handled += 1,
6210                ManagedPollOutcome::Stop => return Ok(handled),
6211                ManagedPollOutcome::Idle => {}
6212            }
6213        }
6214        Ok(handled)
6215    }
6216
6217    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6218        let poll_request_id = unique_request_id("rust-workflow-poll");
6219        let response = self
6220            .retry_worker_operation(|| {
6221                self.client.poll_workflow_task_response_with_request_id(
6222                    &self.worker_id,
6223                    &self.task_queue,
6224                    self.poll_timeout,
6225                    &poll_request_id,
6226                    0,
6227                )
6228            })
6229            .await?;
6230        if response.outcome().should_stop() {
6231            return Ok(ManagedPollOutcome::Stop);
6232        }
6233        let memo_updates_supported =
6234            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6235        let Some(task) = response.task else {
6236            return Ok(ManagedPollOutcome::Idle);
6237        };
6238
6239        let task_id = task.task_id.clone();
6240        let attempt = task.workflow_task_attempt;
6241        let run_id = task.run_id.clone();
6242        let lease_owner = task
6243            .lease_owner
6244            .clone()
6245            .unwrap_or_else(|| self.worker_id.clone());
6246
6247        match self.execute_workflow_task(task) {
6248            Ok(commands)
6249                if commands_use_workflow_memo_updates(&commands) && !memo_updates_supported =>
6250            {
6251                self.client
6252                    .fail_workflow_task(
6253                        &task_id,
6254                        &lease_owner,
6255                        attempt,
6256                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
6257                    )
6258                    .await?;
6259            }
6260            Ok(commands) if commands.is_empty() => {
6261                // A replay can consume a recorded pending durable command
6262                // without producing a new command. The standalone protocol
6263                // acknowledges that state through the typed waiting outcome;
6264                // an empty completion is rejected by servers that require at
6265                // least one executable command.
6266                self.client
6267                    .fail_workflow_task_with_type(
6268                        &task_id,
6269                        &lease_owner,
6270                        attempt,
6271                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6272                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6273                    )
6274                    .await?;
6275            }
6276            Ok(commands) => {
6277                let completion = self
6278                    .client
6279                    .complete_workflow_task(&task_id, &lease_owner, attempt, commands)
6280                    .await;
6281                if let Err(error) = completion {
6282                    if !workflow_task_completion_is_terminal_timeout(
6283                        &error,
6284                        &task_id,
6285                        attempt,
6286                        run_id.as_deref(),
6287                    ) {
6288                        return Err(error);
6289                    }
6290                }
6291            }
6292            Err(error) => {
6293                self.client
6294                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6295                    .await?;
6296            }
6297        }
6298
6299        Ok(ManagedPollOutcome::Handled)
6300    }
6301
6302    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6303        while !stop.load(Ordering::SeqCst) {
6304            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6305                stop.store(true, Ordering::SeqCst);
6306                break;
6307            }
6308        }
6309
6310        Ok(())
6311    }
6312
6313    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6314        let poll_request_id = unique_request_id("rust-activity-poll");
6315        let response = self
6316            .retry_worker_operation(|| {
6317                self.client.poll_activity_task_response_with_request_id(
6318                    &self.worker_id,
6319                    &self.task_queue,
6320                    self.poll_timeout,
6321                    &poll_request_id,
6322                    0,
6323                )
6324            })
6325            .await?;
6326        if response.outcome().should_stop() {
6327            return Ok(ManagedPollOutcome::Stop);
6328        }
6329        let Some(task) = response.task else {
6330            return Ok(ManagedPollOutcome::Idle);
6331        };
6332
6333        let task_id = task.task_id.clone();
6334        let attempt_id = task
6335            .activity_attempt_id
6336            .clone()
6337            .or(task.attempt_id.clone())
6338            .unwrap_or_default();
6339        let lease_owner = task
6340            .lease_owner
6341            .clone()
6342            .unwrap_or_else(|| self.worker_id.clone());
6343        let codec = task.payload_codec.clone();
6344        let result = self.execute_activity_task(task).await;
6345        match result {
6346            Ok(value) => {
6347                let completion = self
6348                    .client
6349                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6350                    .await;
6351                if let Err(error) = completion {
6352                    if !activity_task_rejection_is_final(&error) {
6353                        return Err(error);
6354                    }
6355                }
6356            }
6357            Err(error) => {
6358                let failure = self
6359                    .client
6360                    .fail_activity_task(
6361                        &task_id,
6362                        &attempt_id,
6363                        &lease_owner,
6364                        error.to_string(),
6365                        false,
6366                    )
6367                    .await;
6368                if let Err(error) = failure {
6369                    if !activity_task_rejection_is_final(&error) {
6370                        return Err(error);
6371                    }
6372                }
6373            }
6374        }
6375
6376        Ok(ManagedPollOutcome::Handled)
6377    }
6378
6379    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6380        while !stop.load(Ordering::SeqCst) {
6381            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6382                stop.store(true, Ordering::SeqCst);
6383                break;
6384            }
6385        }
6386
6387        Ok(())
6388    }
6389
6390    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6391        let poll_request_id = unique_request_id("rust-query-poll");
6392        let response = self
6393            .retry_worker_operation(|| {
6394                self.client.poll_query_task_response_with_request_id(
6395                    &self.worker_id,
6396                    &self.task_queue,
6397                    self.poll_timeout,
6398                    &poll_request_id,
6399                    0,
6400                )
6401            })
6402            .await?;
6403        if response.outcome().should_stop() {
6404            return Ok(ManagedPollOutcome::Stop);
6405        }
6406        let Some(task) = response.task else {
6407            return Ok(ManagedPollOutcome::Idle);
6408        };
6409
6410        let query_task_id = task.query_task_id.clone();
6411        let attempt = task.query_task_attempt;
6412        let lease_owner = task
6413            .lease_owner
6414            .clone()
6415            .unwrap_or_else(|| self.worker_id.clone());
6416        let codec = task.payload_codec.clone();
6417
6418        match self.execute_query_task(task).await {
6419            Ok(value) => {
6420                let result_envelope = match encode_typed_envelope(&value, &codec) {
6421                    Ok(result_envelope) => result_envelope,
6422                    Err(error) => {
6423                        let failure = self
6424                            .client
6425                            .fail_query_task(
6426                                &query_task_id,
6427                                &lease_owner,
6428                                attempt,
6429                                error.to_string(),
6430                                "query_result_encode_failed",
6431                                "QueryResultEncodeFailed",
6432                            )
6433                            .await;
6434                        if let Err(error) = failure {
6435                            if !query_task_rejection_is_final(&error) {
6436                                return Err(error);
6437                            }
6438                        }
6439                        return Ok(ManagedPollOutcome::Handled);
6440                    }
6441                };
6442
6443                if let Err(error) = self
6444                    .client
6445                    .complete_query_task_with_envelope(
6446                        &query_task_id,
6447                        &lease_owner,
6448                        attempt,
6449                        value.clone().into_json()?,
6450                        result_envelope,
6451                    )
6452                    .await
6453                {
6454                    if !query_task_rejection_is_final(&error) {
6455                        return Err(error);
6456                    }
6457                }
6458            }
6459            Err(failure) => {
6460                let result = self
6461                    .client
6462                    .fail_query_task(
6463                        &query_task_id,
6464                        &lease_owner,
6465                        attempt,
6466                        failure.message,
6467                        failure.reason,
6468                        failure.failure_type,
6469                    )
6470                    .await;
6471                if let Err(error) = result {
6472                    if !query_task_rejection_is_final(&error) {
6473                        return Err(error);
6474                    }
6475                }
6476            }
6477        }
6478
6479        Ok(ManagedPollOutcome::Handled)
6480    }
6481
6482    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6483        while !stop.load(Ordering::SeqCst) {
6484            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
6485                stop.store(true, Ordering::SeqCst);
6486                break;
6487            }
6488        }
6489
6490        Ok(())
6491    }
6492
6493    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
6494    where
6495        F: FnMut() -> Fut,
6496        Fut: Future<Output = Result<T>>,
6497    {
6498        let mut retries = 0;
6499
6500        loop {
6501            match operation().await {
6502                Err(error)
6503                    if worker_operation_is_retryable(&error)
6504                        && retries < self.retry_policy.max_retries =>
6505                {
6506                    retries += 1;
6507                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
6508                }
6509                result => return result,
6510            }
6511        }
6512    }
6513
6514    async fn execute_query_task(
6515        &self,
6516        mut task: QueryTask,
6517    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
6518        validate_query_task_payloads(&task).map_err(|error| {
6519            QueryTaskExecutionFailure::new(
6520                "query_payload_decode_failed",
6521                error.to_string(),
6522                "QueryPayloadDecodeFailed",
6523            )
6524        })?;
6525
6526        if !self.workflows.contains_key(&task.workflow_type) {
6527            return Err(QueryTaskExecutionFailure::new(
6528                "query_workflow_type_not_registered",
6529                format!("no workflow registered for type {:?}", task.workflow_type),
6530                "WorkflowTypeNotRegistered",
6531            ));
6532        }
6533
6534        let Some(handlers) = self.queries.get(&task.workflow_type) else {
6535            return Err(QueryTaskExecutionFailure::new(
6536                "query_handler_unavailable",
6537                format!(
6538                    "query handlers are unavailable for workflow type {:?}",
6539                    task.workflow_type
6540                ),
6541                "QueryHandlerUnavailable",
6542            ));
6543        };
6544        let Some(query) = handlers.get(&task.query_name) else {
6545            return Err(QueryTaskExecutionFailure::new(
6546                "rejected_unknown_query",
6547                format!("unknown query {:?}", task.query_name),
6548                "QueryFailed",
6549            ));
6550        };
6551
6552        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
6553            .map_err(|error| {
6554            QueryTaskExecutionFailure::new(
6555                "query_payload_decode_failed",
6556                format!("cannot decode query arguments: {error}"),
6557                "QueryPayloadDecodeFailed",
6558            )
6559        })?;
6560        let workflow_input_typed =
6561            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
6562                .map_err(|error| {
6563                    QueryTaskExecutionFailure::new(
6564                        "query_workflow_state_unavailable",
6565                        format!("cannot decode workflow start input: {error}"),
6566                        "QueryWorkflowStateUnavailable",
6567                    )
6568                })?;
6569        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
6570            QueryTaskExecutionFailure::new(
6571                "query_workflow_state_unavailable",
6572                format!("cannot project workflow start input: {error}"),
6573                "QueryWorkflowStateUnavailable",
6574            )
6575        })?;
6576        hydrate_query_history_from_export(&mut task).map_err(|error| {
6577            QueryTaskExecutionFailure::new(
6578                "query_workflow_state_unavailable",
6579                format!("cannot restore query history snapshot: {error}"),
6580                "QueryWorkflowStateUnavailable",
6581            )
6582        })?;
6583        enrich_query_history_from_export(&mut task).map_err(|error| {
6584            QueryTaskExecutionFailure::new(
6585                "query_workflow_state_unavailable",
6586                format!("cannot restore compact query history payloads: {error}"),
6587                "QueryWorkflowStateUnavailable",
6588            )
6589        })?;
6590        let signal_events = query_signal_events(&task).map_err(|error| {
6591            QueryTaskExecutionFailure::new(
6592                "query_workflow_state_unavailable",
6593                format!("cannot decode committed workflow signals: {error}"),
6594                "QueryWorkflowStateUnavailable",
6595            )
6596        })?;
6597        let history_events = Arc::new(std::mem::take(&mut task.history_events));
6598        let context = QueryContext {
6599            workflow_id: task.workflow_id,
6600            run_id: task.run_id,
6601            workflow_type: task.workflow_type.clone(),
6602            run_status: task.run_status,
6603            workflow_input,
6604            workflow_input_avro_value: workflow_input_typed.clone(),
6605            history_events: Arc::clone(&history_events),
6606            signal_events: Arc::new(signal_events),
6607        };
6608
6609        let future = match query {
6610            RegisteredQuery::Snapshot(handler) => handler(context, args),
6611            RegisteredQuery::Replayed {
6612                state_type,
6613                handler,
6614            } => {
6615                let workflow = self
6616                    .workflows
6617                    .get(&task.workflow_type)
6618                    .expect("workflow registration was checked above");
6619                if workflow.state_type != Some(*state_type) {
6620                    return Err(QueryTaskExecutionFailure::new(
6621                        "query_workflow_state_unavailable",
6622                        "replayed query state type does not match its workflow registration",
6623                        "QueryWorkflowStateUnavailable",
6624                    ));
6625                }
6626                let replay = workflow.replay.as_ref().ok_or_else(|| {
6627                    QueryTaskExecutionFailure::new(
6628                        "query_workflow_state_unavailable",
6629                        format!(
6630                            "workflow type {:?} is not registered for instance-state replay",
6631                            task.workflow_type
6632                        ),
6633                        "QueryWorkflowStateUnavailable",
6634                    )
6635                })?;
6636                let workflow_state = Arc::new(Mutex::new(
6637                    WorkflowState::new_with_identity(
6638                        history_events.as_ref().clone(),
6639                        context.workflow_id.clone(),
6640                        context.run_id.clone(),
6641                        self.task_queue.clone(),
6642                        task.payload_codec,
6643                        None,
6644                    )
6645                    .map_err(|error| {
6646                        QueryTaskExecutionFailure::new(
6647                            "query_workflow_state_unavailable",
6648                            format!("workflow replay failed before query: {error}"),
6649                            "QueryWorkflowStateUnavailable",
6650                        )
6651                    })?,
6652                ));
6653                let workflow_context = WorkflowContext {
6654                    state: workflow_state,
6655                };
6656                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
6657                let mut cx = TaskContext::from_waker(noop_waker_ref());
6658                match invocation.future.as_mut().poll(&mut cx) {
6659                    Poll::Ready(Ok(_)) => {
6660                        workflow_context
6661                            .ensure_history_consumed()
6662                            .map_err(|error| {
6663                                QueryTaskExecutionFailure::new(
6664                                    "query_workflow_state_unavailable",
6665                                    format!("workflow replay failed before query: {error}"),
6666                                    "QueryWorkflowStateUnavailable",
6667                                )
6668                            })?;
6669                    }
6670                    Poll::Ready(Err(error)) => {
6671                        return Err(QueryTaskExecutionFailure::new(
6672                            "query_workflow_state_unavailable",
6673                            format!("workflow replay failed before query: {error}"),
6674                            "QueryWorkflowStateUnavailable",
6675                        ));
6676                    }
6677                    Poll::Pending => {
6678                        let commands = workflow_context.take_commands().map_err(|error| {
6679                            QueryTaskExecutionFailure::new(
6680                                "query_workflow_state_unavailable",
6681                                format!("workflow replay failed before query: {error}"),
6682                                "QueryWorkflowStateUnavailable",
6683                            )
6684                        })?;
6685                        if commands.is_empty()
6686                            && !workflow_context
6687                                .matched_recorded_pending()
6688                                .map_err(|error| {
6689                                    QueryTaskExecutionFailure::new(
6690                                        "query_workflow_state_unavailable",
6691                                        format!("workflow replay failed before query: {error}"),
6692                                        "QueryWorkflowStateUnavailable",
6693                                    )
6694                                })?
6695                        {
6696                            return Err(QueryTaskExecutionFailure::new(
6697                                "query_workflow_state_unavailable",
6698                                "workflow replay yielded without a durable command",
6699                                "QueryWorkflowStateUnavailable",
6700                            ));
6701                        }
6702                    }
6703                }
6704                let state = (invocation.snapshot)().map_err(|error| {
6705                    QueryTaskExecutionFailure::new(
6706                        "query_workflow_state_unavailable",
6707                        format!("cannot snapshot replayed workflow state: {error}"),
6708                        "QueryWorkflowStateUnavailable",
6709                    )
6710                })?;
6711                handler(context, state, args).map_err(|message| {
6712                    QueryTaskExecutionFailure::new(
6713                        "query_workflow_state_unavailable",
6714                        message,
6715                        "QueryWorkflowStateUnavailable",
6716                    )
6717                })?
6718            }
6719        };
6720
6721        future.await.map_err(|error| {
6722            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
6723        })
6724    }
6725
6726    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
6727        validate_workflow_task_payloads(&task)?;
6728
6729        if let Some(update_id) = task
6730            .workflow_update_id
6731            .as_deref()
6732            .filter(|update_id| !update_id.is_empty())
6733        {
6734            return self.execute_update_task(&task, update_id);
6735        }
6736
6737        let workflow = self
6738            .workflows
6739            .get(&task.workflow_type)
6740            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
6741        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
6742        let resume_signal = decode_resume_signal(&task)?;
6743        let history_budget = WorkflowHistoryBudget {
6744            event_count: task
6745                .total_history_events
6746                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
6747            size_bytes: task.history_size_bytes,
6748            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
6749            pressure: task.history_budget_pressure.clone(),
6750        };
6751        let workflow_command_identity = task
6752            .workflow_command_id
6753            .clone()
6754            .filter(|identity| !identity.is_empty())
6755            .unwrap_or_default();
6756        let mut workflow_state = WorkflowState::new_with_identity(
6757            task.history_events,
6758            task.workflow_id,
6759            task.run_id,
6760            self.task_queue.clone(),
6761            task.payload_codec.clone(),
6762            resume_signal,
6763        )?;
6764        workflow_state.history_budget = history_budget;
6765        workflow_state.workflow_command_identity = workflow_command_identity;
6766        workflow_state.cancel_requested = task.cancel_requested;
6767        let state = Arc::new(Mutex::new(workflow_state));
6768        let ctx = WorkflowContext { state };
6769        let mut future = (workflow.execute)(ctx.clone(), input);
6770        let mut cx = TaskContext::from_waker(noop_waker_ref());
6771
6772        match future.as_mut().poll(&mut cx) {
6773            Poll::Ready(Ok(result)) => {
6774                ctx.ensure_history_consumed()?;
6775                let result = encode_typed_envelope(&result, &task.payload_codec)?;
6776                let mut commands = ctx.take_commands()?;
6777                commands.push(json!({
6778                    "type": "complete_workflow",
6779                    "result": result
6780                }));
6781                Ok(commands)
6782            }
6783            Poll::Ready(Err(error)) => {
6784                if let Error::ContinueAsNew(request) = error {
6785                    let mut commands = ctx.take_commands()?;
6786                    if let Some(command) = ctx.continue_as_new_command(request)? {
6787                        commands.push(command);
6788                    }
6789                    ctx.ensure_history_consumed()?;
6790                    return Ok(commands);
6791                }
6792                if workflow_task_integrity_error(&error) {
6793                    // Replay and protocol failures describe the workflow-task
6794                    // decision itself. Preserve their specific failure reason
6795                    // instead of replacing it with the derivative fact that
6796                    // recorded commands remain unconsumed.
6797                    return Err(error);
6798                }
6799                // A handler error must not hide a committed durable command that
6800                // upgraded workflow code no longer consumes.
6801                ctx.ensure_history_consumed()?;
6802                let mut commands = ctx.take_commands()?;
6803                commands.push(workflow_failure_command(&error));
6804                Ok(commands)
6805            }
6806            Poll::Pending => {
6807                let commands = ctx.take_commands()?;
6808                if commands.is_empty() && !ctx.matched_recorded_pending()? {
6809                    Err(Error::WorkflowYieldedWithoutCommand)
6810                } else {
6811                    Ok(commands)
6812                }
6813            }
6814        }
6815    }
6816
6817    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
6818        if !self.workflows.contains_key(&task.workflow_type) {
6819            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
6820        }
6821
6822        let accepted = task.history_events.iter().rev().find_map(|event| {
6823            (event.event_type == "UpdateAccepted"
6824                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
6825            .then_some(&event.payload)
6826        });
6827        let update_name = accepted
6828            .and_then(|payload| payload.get("update_name"))
6829            .and_then(Value::as_str)
6830            .or(task.update_name.as_deref())
6831            .unwrap_or_default();
6832        let Some(handler) = self
6833            .updates
6834            .get(&task.workflow_type)
6835            .and_then(|handlers| handlers.get(update_name))
6836        else {
6837            return Ok(vec![json!({
6838                "type": "fail_update",
6839                "update_id": update_id,
6840                "message": format!(
6841                    "no update handler is registered for {}.{update_name}",
6842                    task.workflow_type
6843                ),
6844                "exception_type": "UnknownUpdate",
6845                "non_retryable": true,
6846            })]);
6847        };
6848        let arguments = accepted
6849            .and_then(|payload| payload.get("arguments"))
6850            .or(task.arguments.as_ref());
6851        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
6852        let context = QueryContext {
6853            workflow_id: task.workflow_id.clone(),
6854            run_id: task.run_id.clone(),
6855            workflow_type: task.workflow_type.clone(),
6856            run_status: Some("running".to_string()),
6857            workflow_input: Value::Null,
6858            workflow_input_avro_value: AvroValue::Null,
6859            history_events: Arc::new(task.history_events.clone()),
6860            signal_events: Arc::new(Vec::new()),
6861        };
6862        let mut future = handler(context, arguments);
6863        let mut cx = TaskContext::from_waker(noop_waker_ref());
6864
6865        match future.as_mut().poll(&mut cx) {
6866            Poll::Ready(Ok(result)) => Ok(vec![json!({
6867                "type": "complete_update",
6868                "update_id": update_id,
6869                "result": encode_typed_envelope(&result, &task.payload_codec)?,
6870            })]),
6871            Poll::Ready(Err(error)) => Ok(vec![json!({
6872                "type": "fail_update",
6873                "update_id": update_id,
6874                "message": error.to_string(),
6875                "exception_type": "UpdateFailed",
6876                "non_retryable": true,
6877            })]),
6878            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
6879        }
6880    }
6881
6882    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
6883        validate_activity_task_payloads(&task)?;
6884
6885        let handler = self
6886            .activities
6887            .get(&task.activity_type)
6888            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
6889        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
6890        let attempt_id = task
6891            .activity_attempt_id
6892            .clone()
6893            .or(task.attempt_id.clone())
6894            .unwrap_or_default();
6895        let lease_owner = task
6896            .lease_owner
6897            .clone()
6898            .unwrap_or_else(|| self.worker_id.clone());
6899        let ctx = ActivityContext {
6900            client: self.client.clone(),
6901            task_id: task.task_id,
6902            activity_attempt_id: attempt_id,
6903            lease_owner,
6904            activity_type: task.activity_type,
6905            attempt_number: task.attempt_number,
6906            task_queue: self.task_queue.clone(),
6907            worker_id: self.worker_id.clone(),
6908        };
6909
6910        handler(ctx, args).await
6911    }
6912}
6913
6914fn poller_result(
6915    kind: &str,
6916    result: std::result::Result<Result<()>, tokio::task::JoinError>,
6917) -> Result<()> {
6918    match result {
6919        Ok(result) => result,
6920        Err(error) => Err(Error::WorkerLoop(format!(
6921            "{kind} poller join error: {error}"
6922        ))),
6923    }
6924}
6925
6926fn optional_poller_result(
6927    kind: &str,
6928    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
6929) -> Result<()> {
6930    match result {
6931        Some(result) => poller_result(kind, result),
6932        None => Ok(()),
6933    }
6934}
6935
6936async fn join_pollers(
6937    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
6938    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
6939    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
6940) -> Result<()> {
6941    let mut first_error = None;
6942
6943    if let Some(handle) = workflow_poller {
6944        if let Err(error) = poller_result("workflow", handle.await) {
6945            first_error.get_or_insert(error);
6946        }
6947    }
6948
6949    if let Some(handle) = activity_poller {
6950        if let Err(error) = poller_result("activity", handle.await) {
6951            first_error.get_or_insert(error);
6952        }
6953    }
6954
6955    if let Some(handle) = query_poller {
6956        if let Err(error) = poller_result("query", handle.await) {
6957            first_error.get_or_insert(error);
6958        }
6959    }
6960
6961    if let Some(error) = first_error {
6962        Err(error)
6963    } else {
6964        Ok(())
6965    }
6966}
6967
6968fn default_worker_id() -> String {
6969    let millis = SystemTime::now()
6970        .duration_since(UNIX_EPOCH)
6971        .unwrap_or_default()
6972        .as_millis();
6973    format!("rust-worker-{}-{millis}", std::process::id())
6974}
6975
6976fn percent_encode_path_segment(segment: &str) -> String {
6977    const HEX: &[u8; 16] = b"0123456789ABCDEF";
6978    let mut encoded = String::with_capacity(segment.len());
6979
6980    for byte in segment.bytes() {
6981        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
6982            encoded.push(char::from(byte));
6983        } else {
6984            encoded.push('%');
6985            encoded.push(char::from(HEX[(byte >> 4) as usize]));
6986            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
6987        }
6988    }
6989
6990    encoded
6991}
6992
6993fn unique_request_id(prefix: &str) -> String {
6994    let nanos = SystemTime::now()
6995        .duration_since(UNIX_EPOCH)
6996        .unwrap_or_default()
6997        .as_nanos();
6998    format!("{prefix}-{}-{nanos}", std::process::id())
6999}
7000
7001#[derive(Debug)]
7002struct QueryTaskExecutionFailure {
7003    reason: String,
7004    message: String,
7005    failure_type: String,
7006}
7007
7008impl QueryTaskExecutionFailure {
7009    fn new(
7010        reason: impl Into<String>,
7011        message: impl Into<String>,
7012        failure_type: impl Into<String>,
7013    ) -> Self {
7014        Self {
7015            reason: reason.into(),
7016            message: message.into(),
7017            failure_type: failure_type.into(),
7018        }
7019    }
7020}
7021
7022/// Typed local state owned by one deterministic workflow invocation.
7023///
7024/// Use [`WorkflowInstance::update`] for the same state transitions during
7025/// ordinary execution and replay. A replayed query receives a detached
7026/// immutable `Arc<S>` rather than this mutation-capable handle.
7027#[derive(Clone, Debug)]
7028pub struct WorkflowInstance<S> {
7029    state: Arc<Mutex<S>>,
7030}
7031
7032impl<S> WorkflowInstance<S> {
7033    fn new(state: S) -> Self {
7034        Self {
7035            state: Arc::new(Mutex::new(state)),
7036        }
7037    }
7038
7039    /// Read the current workflow-instance state without changing it.
7040    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7041        let state = self
7042            .state
7043            .lock()
7044            .map_err(|_| Error::WorkflowStatePoisoned)?;
7045        Ok(reader(&state))
7046    }
7047
7048    /// Apply one deterministic workflow-instance state transition.
7049    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7050        let mut state = self
7051            .state
7052            .lock()
7053            .map_err(|_| Error::WorkflowStatePoisoned)?;
7054        Ok(transition(&mut state))
7055    }
7056}
7057
7058impl<S: Clone> WorkflowInstance<S> {
7059    fn snapshot(&self) -> Result<S> {
7060        self.read(Clone::clone)
7061    }
7062}
7063
7064#[derive(Clone, Debug)]
7065pub struct WorkflowContext {
7066    state: Arc<Mutex<WorkflowState>>,
7067}
7068
7069fn valid_memo_key(key: &str) -> bool {
7070    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7071
7072    !key.is_empty()
7073        && key.len() <= 64
7074        && (numeric_candidate.is_empty()
7075            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7076        && key
7077            .bytes()
7078            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7079}
7080
7081fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7082    BASE64
7083        .decode(encode_avro_value(value)?.blob)
7084        .map(|bytes| bytes.len())
7085        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7086}
7087
7088fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7089    let AvroValue::Map(entries) = value else {
7090        return Err(Error::InvalidMemoUpdate(
7091            "entries must serialize to an Avro string-keyed map".to_string(),
7092        ));
7093    };
7094    if require_entries && entries.is_empty() {
7095        return Err(Error::InvalidMemoUpdate(
7096            "at least one entry is required".to_string(),
7097        ));
7098    }
7099    if entries.len() > MAX_MEMO_ENTRIES {
7100        return Err(Error::InvalidMemoUpdate(format!(
7101            "at most {MAX_MEMO_ENTRIES} entries are allowed"
7102        )));
7103    }
7104
7105    for (key, value) in &entries {
7106        if !valid_memo_key(&key) {
7107            return Err(Error::InvalidMemoUpdate(
7108                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7109            ));
7110        }
7111        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7112            return Err(Error::InvalidMemoUpdate(format!(
7113                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7114            )));
7115        }
7116    }
7117
7118    let value = AvroValue::Map(entries);
7119    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7120        return Err(Error::InvalidMemoUpdate(format!(
7121            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7122        )));
7123    }
7124    Ok(value)
7125}
7126
7127fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7128    let object = envelope.as_object().ok_or_else(|| {
7129        Error::InvalidMemoUpdate(
7130            "history field must use the public {codec, blob} payload envelope".to_string(),
7131        )
7132    })?;
7133    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7134        return Err(Error::InvalidMemoUpdate(
7135            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7136        ));
7137    }
7138
7139    canonical_memo_entries(
7140        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7141        require_entries,
7142    )
7143}
7144
7145impl WorkflowContext {
7146    /// Identity of the parent workflow currently being replayed.
7147    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7148        let state = self
7149            .state
7150            .lock()
7151            .map_err(|_| Error::WorkflowStatePoisoned)?;
7152        Ok(WorkflowIdentity {
7153            workflow_id: state.workflow_id.clone(),
7154            run_id: state.run_id.clone(),
7155        })
7156    }
7157
7158    /// Return the server-published history budget for this workflow task.
7159    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
7160        let state = self
7161            .state
7162            .lock()
7163            .map_err(|_| Error::WorkflowStatePoisoned)?;
7164        Ok(state.history_budget.clone())
7165    }
7166
7167    /// Continue this workflow instance as a fresh run with replacement arguments.
7168    ///
7169    /// Return this value directly from the workflow handler. The worker converts
7170    /// it to the terminal protocol command only after replay has consumed every
7171    /// recorded durable command.
7172    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
7173        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
7174    }
7175
7176    /// Continue as new with optional workflow-type and task-queue overrides.
7177    pub fn continue_as_new_with_options<T: Serialize>(
7178        &self,
7179        options: ContinueAsNewOptions,
7180        args: T,
7181    ) -> Result<Value> {
7182        options.validate()?;
7183        Err(Error::ContinueAsNew(ContinueAsNewRequest {
7184            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
7185            options,
7186        }))
7187    }
7188
7189    pub fn activity<T: Serialize>(
7190        &self,
7191        activity_type: impl Into<String>,
7192        args: T,
7193    ) -> ActivityCall {
7194        self.activity_with_options(activity_type, ActivityOptions::new(), args)
7195    }
7196
7197    pub fn activity_on_queue<T, Q>(
7198        &self,
7199        activity_type: impl Into<String>,
7200        task_queue: Option<Q>,
7201        args: T,
7202    ) -> ActivityCall
7203    where
7204        T: Serialize,
7205        Q: Into<String>,
7206    {
7207        let mut options = ActivityOptions::new();
7208        options.task_queue = task_queue.map(Into::into);
7209        self.activity_with_options(activity_type, options, args)
7210    }
7211
7212    /// Schedule one durable activity with retry, routing, and timeout options.
7213    ///
7214    /// Options are validated before the command is emitted. Once the command is
7215    /// recorded, replay consumes the same activity lifecycle at this command
7216    /// position and never emits a duplicate schedule.
7217    ///
7218    /// ```no_run
7219    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
7220    /// # use std::time::Duration;
7221    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
7222    /// let result = ctx
7223    ///     .activity_with_options(
7224    ///         "charge-card",
7225    ///         ActivityOptions::new()
7226    ///             .task_queue("payments")
7227    ///             .retry_policy(
7228    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
7229    ///                     Duration::from_secs(1),
7230    ///                     2,
7231    ///                     Some(Duration::from_secs(30)),
7232    ///                 ),
7233    ///             )
7234    ///             .start_to_close_timeout(Duration::from_secs(60))
7235    ///             .schedule_to_close_timeout(Duration::from_secs(180))
7236    ///             .heartbeat_timeout(Duration::from_secs(15)),
7237    ///         json!([{"order_id": "order-42"}]),
7238    ///     )
7239    ///     .await;
7240    /// match result {
7241    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
7242    ///         "reason": failure.reason,
7243    ///         "timeout_kind": failure.timeout_kind,
7244    ///     })),
7245    ///     other => other,
7246    /// }
7247    /// # }
7248    /// ```
7249    pub fn activity_with_options<T: Serialize>(
7250        &self,
7251        activity_type: impl Into<String>,
7252        options: ActivityOptions,
7253        args: T,
7254    ) -> ActivityCall {
7255        ActivityCall {
7256            ctx: self.clone(),
7257            activity_type: activity_type.into(),
7258            options,
7259            args: Some(AvroValue::from_serialize(&args)),
7260            scheduled: false,
7261            parallel_group_path: Vec::new(),
7262        }
7263    }
7264
7265    pub async fn activity_avro_value<T: Serialize>(
7266        &self,
7267        activity_type: impl Into<String>,
7268        args: T,
7269    ) -> Result<AvroValue> {
7270        let mut call = self.activity(activity_type, args);
7271        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7272    }
7273
7274    pub async fn activity_avro_value_with_options<T: Serialize>(
7275        &self,
7276        activity_type: impl Into<String>,
7277        options: ActivityOptions,
7278        args: T,
7279    ) -> Result<AvroValue> {
7280        let mut call = self.activity_with_options(activity_type, options, args);
7281        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7282    }
7283
7284    /// Schedule an activity with a Serde request and decode its Serde result.
7285    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
7286    where
7287        I: Serialize,
7288        O: DeserializeOwned,
7289    {
7290        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
7291            .await
7292    }
7293
7294    /// Schedule an activity with options and decode its result into `O`.
7295    ///
7296    /// Both directions use the fixed Avro Value codec. In particular, this
7297    /// method does not deserialize the JSON-safe inspection projection returned
7298    /// by the dynamic [`ActivityCall`] future.
7299    pub async fn activity_typed_with_options<I, O>(
7300        &self,
7301        activity_type: impl Into<String>,
7302        options: ActivityOptions,
7303        args: I,
7304    ) -> Result<O>
7305    where
7306        I: Serialize,
7307        O: DeserializeOwned,
7308    {
7309        let activity_type = activity_type.into();
7310        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
7311            handler_type_error::<I>(
7312                HandlerKind::Activity,
7313                &activity_type,
7314                HandlerValueKind::Input,
7315                error.to_string(),
7316            )
7317        });
7318        let mut call = ActivityCall {
7319            ctx: self.clone(),
7320            activity_type: activity_type.clone(),
7321            options,
7322            args: Some(encoded),
7323            scheduled: false,
7324            parallel_group_path: Vec::new(),
7325        };
7326        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
7327        decode_handler_result(result, HandlerKind::Activity, &activity_type)
7328    }
7329
7330    /// Schedule and join a deterministic activity/child/timer group.
7331    ///
7332    /// Nested groups retain their input shape. Every durable leaf is scheduled
7333    /// before this future yields, results are assembled by declaration order,
7334    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
7335    /// declaration path, stable group metadata, and completed siblings.
7336    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7337        ParallelCall::new(self.clone(), operations)
7338    }
7339
7340    /// Alias for [`WorkflowContext::parallel`].
7341    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
7342        self.parallel(operations)
7343    }
7344
7345    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
7346    pub async fn parallel_avro_value(
7347        &self,
7348        operations: Vec<ParallelOperation>,
7349    ) -> Result<Vec<ParallelAvroResult>> {
7350        let mut call = self.parallel(operations);
7351        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7352    }
7353
7354    /// Create a workflow-local deterministic compensation registry.
7355    pub fn saga(&self) -> Saga {
7356        Saga::new(self.clone())
7357    }
7358
7359    /// Whether the current workflow task carries a cooperative cancel request.
7360    pub fn is_cancellation_requested(&self) -> Result<bool> {
7361        let state = self
7362            .state
7363            .lock()
7364            .map_err(|_| Error::WorkflowStatePoisoned)?;
7365        Ok(state.cancel_requested)
7366    }
7367
7368    /// Raise a typed cooperative cancellation at an author-controlled point.
7369    ///
7370    /// Passing this result to [`Saga::finish`] compensates already registered
7371    /// forward steps before the cancellation remains the initiating outcome.
7372    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
7373        if self.is_cancellation_requested()? {
7374            return Err(Error::WorkflowCancellationRequested(
7375                WorkflowCancellationRequested,
7376            ));
7377        }
7378        Ok(())
7379    }
7380
7381    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
7382        SignalCall {
7383            ctx: self.clone(),
7384            signal_name: signal_name.into(),
7385            opened_wait: false,
7386            matched_pending: false,
7387        }
7388    }
7389
7390    pub async fn wait_signal_avro_value(
7391        &self,
7392        signal_name: impl Into<String>,
7393    ) -> Result<Vec<AvroValue>> {
7394        let mut call = self.wait_signal(signal_name);
7395        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
7396    }
7397
7398    /// Return every committed signal argument list with the given name.
7399    ///
7400    /// This history-backed view is deterministic and is intended for
7401    /// condition predicates that must be re-evaluated after a signal while the
7402    /// workflow is blocked on [`WorkflowContext::wait_condition`].
7403    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
7404        self.signals_avro_value(signal_name)?
7405            .into_iter()
7406            .map(|arguments| {
7407                arguments
7408                    .into_iter()
7409                    .map(AvroValue::into_json)
7410                    .collect::<Result<Vec<_>>>()
7411            })
7412            .collect()
7413    }
7414
7415    /// Lossless fixed Avro Value view of committed signals with the given name.
7416    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
7417        let state = self
7418            .state
7419            .lock()
7420            .map_err(|_| Error::WorkflowStatePoisoned)?;
7421        state
7422            .history_events
7423            .iter()
7424            .filter(|event| {
7425                event.event_type == "SignalReceived"
7426                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
7427            })
7428            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
7429            .collect()
7430    }
7431
7432    /// Return every committed update argument list with the given name.
7433    ///
7434    /// Accepted and applied records for the same update ID are de-duplicated.
7435    /// A Server task created after an update therefore replays the workflow and
7436    /// re-evaluates an open condition without application polling.
7437    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
7438        self.updates_avro_value(update_name)?
7439            .into_iter()
7440            .map(|arguments| {
7441                arguments
7442                    .into_iter()
7443                    .map(AvroValue::into_json)
7444                    .collect::<Result<Vec<_>>>()
7445            })
7446            .collect()
7447    }
7448
7449    /// Lossless fixed Avro Value view of committed updates with the given name.
7450    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
7451        let state = self
7452            .state
7453            .lock()
7454            .map_err(|_| Error::WorkflowStatePoisoned)?;
7455        let mut seen = Vec::new();
7456        let mut updates = Vec::new();
7457        for event in state.history_events.iter() {
7458            if !matches!(
7459                event.event_type.as_str(),
7460                "UpdateAccepted" | "UpdateApplied"
7461            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
7462                || event.payload.get("arguments").is_none()
7463            {
7464                continue;
7465            }
7466            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
7467                if seen.iter().any(|recorded| recorded == update_id) {
7468                    continue;
7469                }
7470                seen.push(update_id.to_string());
7471            }
7472            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
7473        }
7474        Ok(updates)
7475    }
7476
7477    /// Wait for a deterministic predicate to become true or for its durable
7478    /// timeout to elapse.
7479    ///
7480    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
7481    /// predicate tokens automatically change the recorded definition
7482    /// fingerprint. Direct callers must provide an equally stable identity in
7483    /// [`ConditionWaitOptions`].
7484    pub fn wait_condition<F>(
7485        &self,
7486        options: ConditionWaitOptions,
7487        predicate: F,
7488    ) -> ConditionWaitCall
7489    where
7490        F: Fn() -> Result<bool> + Send + 'static,
7491    {
7492        ConditionWaitCall {
7493            ctx: self.clone(),
7494            options,
7495            predicate: Box::new(predicate),
7496            opened_wait: false,
7497        }
7498    }
7499
7500    /// Wait for server-backed durable time without blocking the worker executor.
7501    ///
7502    /// Polling this future emits one `start_timer` command and yields. The
7503    /// server records the deadline, so neither worker nor server restarts reset
7504    /// the wait. Replay resolves the future only from a `TimerScheduled` and
7505    /// `TimerFired` pair at the same position in the shared durable-command
7506    /// stream, with matching sequence, timer identity, and delay. Sub-second
7507    /// durations round up because protocol deadlines use whole seconds.
7508    ///
7509    /// ```no_run
7510    /// # use durable_workflow::{json, Client, Worker};
7511    /// # use std::time::Duration;
7512    /// # fn configure(client: Client) {
7513    /// let mut worker = Worker::new(client, "rust-workers");
7514    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
7515    ///     ctx.sleep(Duration::from_secs(5)).await?;
7516    ///     Ok(json!({"status": "timer fired"}))
7517    /// });
7518    /// # }
7519    /// ```
7520    pub fn sleep(&self, duration: Duration) -> TimerCall {
7521        let delay_seconds = duration
7522            .as_secs()
7523            .checked_add(u64::from(duration.subsec_nanos() > 0));
7524        TimerCall {
7525            ctx: self.clone(),
7526            delay_seconds,
7527            scheduled: false,
7528            matched_pending: false,
7529            parallel_group_path: Vec::new(),
7530        }
7531    }
7532
7533    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
7534    pub fn start_timer(&self, duration: Duration) -> TimerCall {
7535        self.sleep(duration)
7536    }
7537
7538    /// Evaluate a non-deterministic callback once and durably record its typed value.
7539    ///
7540    /// On replay the callback is not invoked: the value is decoded from the
7541    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
7542    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
7543    /// small values that must remain fixed for the lifetime of a workflow run.
7544    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
7545    where
7546        T: Serialize + DeserializeOwned,
7547        F: FnOnce() -> T,
7548    {
7549        {
7550            let mut state = self
7551                .state
7552                .lock()
7553                .map_err(|_| Error::WorkflowStatePoisoned)?;
7554            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7555                return match recorded {
7556                    RecordedCommand::SideEffect { sequence, value } => {
7557                        state.command_cursor += 1;
7558                        value.deserialize().map_err(|error| {
7559                            Error::NonDeterministicReplay(ReplayFailure::new(
7560                                "side_effect_type_mismatch",
7561                                Some(sequence),
7562                                Some(std::any::type_name::<T>().to_string()),
7563                                Some(error.to_string()),
7564                                "recorded side-effect value is incompatible with the requested Rust type",
7565                            ))
7566                        })
7567                    }
7568                    other => Err(command_mismatch(&other, "side effect")),
7569                };
7570            }
7571        }
7572
7573        let value = callback();
7574        let avro_value = AvroValue::from_serialize(&value)?;
7575        let mut state = self
7576            .state
7577            .lock()
7578            .map_err(|_| Error::WorkflowStatePoisoned)?;
7579        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
7580        state.commands.push(json!({
7581            "type": "record_side_effect",
7582            "result": result,
7583        }));
7584        Ok(value)
7585    }
7586
7587    /// Record or replay a lossless fixed Avro Value side effect.
7588    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
7589    where
7590        F: FnOnce() -> AvroValue,
7591    {
7592        {
7593            let mut state = self
7594                .state
7595                .lock()
7596                .map_err(|_| Error::WorkflowStatePoisoned)?;
7597            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7598                return match recorded {
7599                    RecordedCommand::SideEffect { value, .. } => {
7600                        state.command_cursor += 1;
7601                        Ok(value)
7602                    }
7603                    other => Err(command_mismatch(&other, "side effect")),
7604                };
7605            }
7606        }
7607
7608        let value = callback();
7609        let mut state = self
7610            .state
7611            .lock()
7612            .map_err(|_| Error::WorkflowStatePoisoned)?;
7613        let result = encode_typed_envelope(&value, &state.payload_codec)?;
7614        state.commands.push(json!({
7615            "type": "record_side_effect",
7616            "result": result,
7617        }));
7618        Ok(value)
7619    }
7620
7621    /// Append output items at a deterministic workflow command boundary.
7622    ///
7623    /// Stable idempotency keys are derived from the server-provided durable
7624    /// workflow command identity, command ordinal, and item index. Replay
7625    /// consumes the recorded side effect and never emits another append.
7626    pub fn append_workflow_stream(
7627        &self,
7628        stream_name: impl Into<String>,
7629        items: &[WorkflowStreamAppendItem],
7630        max_pending_items: Option<u64>,
7631    ) -> Result<()> {
7632        if items.is_empty() {
7633            return Err(Error::Codec(
7634                "workflow_stream_items_empty: append requires at least one item".to_string(),
7635            ));
7636        }
7637        if max_pending_items == Some(0) {
7638            return Err(Error::Codec(
7639                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
7640                    .to_string(),
7641            ));
7642        }
7643        let stream_name = stream_name.into();
7644        if stream_name.is_empty() {
7645            return Err(Error::Codec(
7646                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
7647            ));
7648        }
7649
7650        let mut state = self
7651            .state
7652            .lock()
7653            .map_err(|_| Error::WorkflowStatePoisoned)?;
7654        let command_ordinal = state.workflow_stream_command_counter;
7655        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7656            state.workflow_stream_command_counter += 1;
7657            return match recorded {
7658                RecordedCommand::SideEffect { .. } => {
7659                    state.command_cursor += 1;
7660                    Ok(())
7661                }
7662                other => Err(command_mismatch(&other, "workflow stream append")),
7663            };
7664        }
7665
7666        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
7667        state.workflow_stream_command_counter += 1;
7668        let wire_items = items
7669            .iter()
7670            .enumerate()
7671            .map(|(item_index, item)| {
7672                item.wire_value(Some(format!(
7673                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
7674                )))
7675            })
7676            .collect::<Vec<_>>();
7677        let mut directive = json!({
7678            "operation": "append",
7679            "stream_name": stream_name,
7680            "command_identity": identity,
7681            "command_ordinal": command_ordinal,
7682            "items": wire_items,
7683        });
7684        if let Some(max_pending_items) = max_pending_items {
7685            directive["max_pending_items"] = json!(max_pending_items);
7686        }
7687        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
7688        state.commands.push(json!({
7689            "type": "record_side_effect",
7690            "result": result,
7691            "workflow_stream": directive,
7692        }));
7693        Ok(())
7694    }
7695
7696    /// Close a run-scoped output stream at a deterministic command boundary.
7697    pub fn close_workflow_stream(
7698        &self,
7699        stream_name: impl Into<String>,
7700        retention_seconds: Option<u64>,
7701    ) -> Result<()> {
7702        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
7703    }
7704
7705    /// Mark a run-scoped output stream errored at a deterministic command boundary.
7706    pub fn error_workflow_stream(
7707        &self,
7708        stream_name: impl Into<String>,
7709        error_reason: impl Into<String>,
7710        retention_seconds: Option<u64>,
7711    ) -> Result<()> {
7712        let error_reason = error_reason.into();
7713        if error_reason.is_empty() {
7714            return Err(Error::Codec(
7715                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
7716            ));
7717        }
7718        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
7719    }
7720
7721    fn finish_workflow_stream(
7722        &self,
7723        stream_name: String,
7724        error_reason: Option<String>,
7725        retention_seconds: Option<u64>,
7726    ) -> Result<()> {
7727        if stream_name.is_empty() {
7728            return Err(Error::Codec(
7729                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
7730            ));
7731        }
7732        if retention_seconds == Some(0) {
7733            return Err(Error::Codec(
7734                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
7735            ));
7736        }
7737        let mut state = self
7738            .state
7739            .lock()
7740            .map_err(|_| Error::WorkflowStatePoisoned)?;
7741        let command_ordinal = state.workflow_stream_command_counter;
7742        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7743            state.workflow_stream_command_counter += 1;
7744            return match recorded {
7745                RecordedCommand::SideEffect { .. } => {
7746                    state.command_cursor += 1;
7747                    Ok(())
7748                }
7749                other => Err(command_mismatch(&other, "workflow stream close")),
7750            };
7751        }
7752        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
7753        state.workflow_stream_command_counter += 1;
7754        let mut directive = json!({
7755            "operation": if error_reason.is_some() { "error" } else { "close" },
7756            "stream_name": stream_name,
7757            "command_identity": identity,
7758            "command_ordinal": command_ordinal,
7759        });
7760        if let Some(error_reason) = error_reason {
7761            directive["error_reason"] = json!(error_reason);
7762        }
7763        if let Some(retention_seconds) = retention_seconds {
7764            directive["retention_seconds"] = json!(retention_seconds);
7765        }
7766        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
7767        state.commands.push(json!({
7768            "type": "record_side_effect",
7769            "result": result,
7770            "workflow_stream": directive,
7771        }));
7772        Ok(())
7773    }
7774
7775    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
7776        let identity = state.workflow_command_identity.as_str();
7777        if identity.is_empty() {
7778            return Err(Error::MissingWorkflowCommandIdentity);
7779        }
7780        Ok(identity)
7781    }
7782
7783    /// Validate, emit, or replay a typed workflow search-attribute update.
7784    ///
7785    /// The command is non-blocking within a workflow decision, but its
7786    /// `SearchAttributesUpserted` event occupies the same deterministic command
7787    /// stream as activities, timers, conditions, and other durable operations.
7788    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
7789        update.validate()?;
7790        let (attributes, attribute_types) = update.into_wire_parts();
7791        let mut state = self
7792            .state
7793            .lock()
7794            .map_err(|_| Error::WorkflowStatePoisoned)?;
7795
7796        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7797            return match recorded {
7798                RecordedCommand::SearchAttributes {
7799                    sequence,
7800                    attributes: recorded_attributes,
7801                    attribute_types: recorded_attribute_types,
7802                } => {
7803                    if recorded_attributes != attributes {
7804                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7805                            "search_attribute_value_mismatch",
7806                            Some(sequence),
7807                            Some(recorded_attributes.to_string()),
7808                            Some(attributes.to_string()),
7809                            "search-attribute values differ from the recorded durable command",
7810                        )));
7811                    }
7812                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
7813                        if recorded_types != attribute_types {
7814                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7815                                "search_attribute_type_mismatch",
7816                                Some(sequence),
7817                                Some(json!(recorded_types).to_string()),
7818                                Some(json!(attribute_types).to_string()),
7819                                "search-attribute declared types differ from the recorded durable command",
7820                            )));
7821                        }
7822                    }
7823                    state.command_cursor += 1;
7824                    Ok(())
7825                }
7826                other => Err(command_mismatch(&other, "search-attribute update")),
7827            };
7828        }
7829
7830        let mut command = serde_json::Map::from_iter([
7831            ("type".to_string(), json!("upsert_search_attributes")),
7832            ("attributes".to_string(), attributes),
7833        ]);
7834        if !attribute_types.is_empty() {
7835            command.insert("attribute_types".to_string(), json!(attribute_types));
7836        }
7837        state.commands.push(Value::Object(command));
7838        Ok(())
7839    }
7840
7841    /// Record a UUIDv4 once and return the same UUID on every replay.
7842    pub fn uuid_v4(&self) -> Result<Uuid> {
7843        self.side_effect(Uuid::new_v4)
7844    }
7845
7846    /// Select the newest supported version for a change, or replay the version
7847    /// already committed for that stable change ID.
7848    pub fn get_version(
7849        &self,
7850        change_id: impl Into<String>,
7851        min_supported: i32,
7852        max_supported: i32,
7853    ) -> Result<i32> {
7854        let change_id = change_id.into();
7855        if change_id.trim().is_empty() {
7856            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7857                "version_change_id_invalid",
7858                None,
7859                Some("non-empty change ID".to_string()),
7860                Some(change_id),
7861                "version markers require a stable non-empty change ID",
7862            )));
7863        }
7864        if min_supported > max_supported {
7865            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7866                "version_range_invalid",
7867                None,
7868                Some("min_supported <= max_supported".to_string()),
7869                Some(format!("{min_supported}..={max_supported}")),
7870                "version marker supported range is invalid",
7871            )));
7872        }
7873
7874        let mut state = self
7875            .state
7876            .lock()
7877            .map_err(|_| Error::WorkflowStatePoisoned)?;
7878        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
7879            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
7880            return Ok(version);
7881        }
7882
7883        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7884            return match recorded {
7885                RecordedCommand::VersionMarker {
7886                    sequence,
7887                    change_id: recorded_change_id,
7888                    version,
7889                    ..
7890                } => {
7891                    if recorded_change_id != change_id {
7892                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7893                            "version_change_id_mismatch",
7894                            Some(sequence),
7895                            Some(recorded_change_id),
7896                            Some(change_id),
7897                            "recorded version marker change ID differs from current workflow code",
7898                        )));
7899                    }
7900                    ensure_version_supported(
7901                        &change_id,
7902                        version,
7903                        min_supported,
7904                        max_supported,
7905                        sequence,
7906                    )?;
7907                    state.command_cursor += 1;
7908                    state.version_markers.insert(change_id, (version, sequence));
7909                    Ok(version)
7910                }
7911                other => Err(command_mismatch(
7912                    &other,
7913                    format!("version marker:{change_id}"),
7914                )),
7915            };
7916        }
7917
7918        let version = max_supported;
7919        state.commands.push(json!({
7920            "type": "record_version_marker",
7921            "change_id": change_id,
7922            "version": version,
7923            "min_supported": min_supported,
7924            "max_supported": max_supported,
7925        }));
7926        // Sequence numbers are assigned by the server. Zero identifies a marker
7927        // selected in this uncommitted decision batch for duplicate-call checks.
7928        state.version_markers.insert(change_id, (version, 0));
7929        Ok(version)
7930    }
7931
7932    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
7933    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
7934        Ok(self.get_version(change_id, -1, 1)? == 1)
7935    }
7936
7937    /// Keep a patch marker in history after the legacy branch has been removed.
7938    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
7939        self.get_version(change_id, -1, 1).map(|_| ())
7940    }
7941
7942    /// Merge non-indexed workflow memo metadata through durable history.
7943    ///
7944    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
7945    /// public Avro payload envelope consumed by Server and Cloud runtimes.
7946    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
7947        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
7948        let mut state = self
7949            .state
7950            .lock()
7951            .map_err(|_| Error::WorkflowStatePoisoned)?;
7952
7953        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
7954            return match recorded {
7955                RecordedCommand::Memo {
7956                    sequence,
7957                    entries: recorded_entries,
7958                } => {
7959                    if recorded_entries != entries {
7960                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
7961                            "memo_update_mismatch",
7962                            Some(sequence),
7963                            Some(format!("{recorded_entries:?}")),
7964                            Some(format!("{entries:?}")),
7965                            "recorded memo entries differ from the current workflow update",
7966                        )));
7967                    }
7968                    state.command_cursor += 1;
7969                    Ok(())
7970                }
7971                other => Err(command_mismatch(&other, "memo upsert")),
7972            };
7973        }
7974
7975        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
7976        state.commands.push(json!({
7977            "type": "upsert_memo",
7978            "entries": entries_envelope,
7979        }));
7980        Ok(())
7981    }
7982
7983    /// Start a named durable child on an explicit queue and await its result.
7984    ///
7985    /// The command is recorded in the parent's sequence-ordered durable command
7986    /// stream. Replay keeps a scheduled child pending without emitting another
7987    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
7988    /// values preserve the history payload codec and include both sides of the
7989    /// durable relationship; failures are returned as
7990    /// [`Error::ChildWorkflowFailed`].
7991    ///
7992    /// ```no_run
7993    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
7994    /// # fn configure(client: Client) {
7995    /// let mut worker = Worker::new(client, "parent-workers");
7996    /// worker.register_workflow("order-parent", |ctx, _input| async move {
7997    ///     let child = ctx
7998    ///         .start_child_workflow(
7999    ///             "fulfil-order",
8000    ///             ChildWorkflowOptions::new("fulfilment-workers")
8001    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
8002    ///             json!([{"order_id": "order-42"}]),
8003    ///         )
8004    ///         .await?;
8005    ///     Ok(child.result)
8006    /// });
8007    /// # }
8008    /// ```
8009    pub fn start_child_workflow<T: Serialize>(
8010        &self,
8011        workflow_type: impl Into<String>,
8012        options: ChildWorkflowOptions,
8013        args: T,
8014    ) -> ChildWorkflowCall {
8015        ChildWorkflowCall {
8016            ctx: self.clone(),
8017            workflow_type: workflow_type.into(),
8018            options,
8019            args: Some(AvroValue::from_serialize(&args)),
8020            scheduled: false,
8021            matched_pending: false,
8022            parallel_group_path: Vec::new(),
8023        }
8024    }
8025
8026    pub async fn start_child_workflow_avro_value<T: Serialize>(
8027        &self,
8028        workflow_type: impl Into<String>,
8029        options: ChildWorkflowOptions,
8030        args: T,
8031    ) -> Result<ChildWorkflowAvroResult> {
8032        let mut call = self.start_child_workflow(workflow_type, options, args);
8033        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8034    }
8035
8036    fn take_commands(&self) -> Result<Vec<Value>> {
8037        let mut state = self
8038            .state
8039            .lock()
8040            .map_err(|_| Error::WorkflowStatePoisoned)?;
8041        Ok(std::mem::take(&mut state.commands))
8042    }
8043
8044    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8045        let mut state = self
8046            .state
8047            .lock()
8048            .map_err(|_| Error::WorkflowStatePoisoned)?;
8049
8050        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8051            return Err(command_mismatch(&recorded, "continue as new"));
8052        }
8053        if state.recorded_continue_as_new_sequence.is_some() {
8054            state.continue_as_new_consumed = true;
8055            return Ok(None);
8056        }
8057
8058        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8059        let mut command = serde_json::Map::from_iter([
8060            ("type".to_string(), json!("continue_as_new")),
8061            ("arguments".to_string(), arguments),
8062            ("queue".to_string(), json!(state.task_queue.clone())),
8063        ]);
8064        if let Some(workflow_type) = request.options.workflow_type {
8065            command.insert("workflow_type".to_string(), json!(workflow_type));
8066        }
8067        if let Some(task_queue) = request.options.task_queue {
8068            command.insert("queue".to_string(), json!(task_queue));
8069        }
8070        Ok(Some(Value::Object(command)))
8071    }
8072
8073    fn matched_recorded_pending(&self) -> Result<bool> {
8074        let state = self
8075            .state
8076            .lock()
8077            .map_err(|_| Error::WorkflowStatePoisoned)?;
8078        Ok(state.matched_recorded_pending)
8079    }
8080
8081    fn ensure_history_consumed(&self) -> Result<()> {
8082        let state = self
8083            .state
8084            .lock()
8085            .map_err(|_| Error::WorkflowStatePoisoned)?;
8086        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8087            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8088                "recorded_commands_unconsumed",
8089                Some(command.sequence()),
8090                Some(command.shape().to_string()),
8091                Some("workflow completion".to_string()),
8092                "workflow completed before consuming all recorded durable commands",
8093            )));
8094        }
8095        if let Some(sequence) = state
8096            .recorded_continue_as_new_sequence
8097            .filter(|_| !state.continue_as_new_consumed)
8098        {
8099            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8100                "recorded_continue_as_new_unconsumed",
8101                Some(sequence),
8102                Some("continue as new".to_string()),
8103                Some("workflow completion".to_string()),
8104                "workflow completed without consuming its recorded continue-as-new transition",
8105            )));
8106        }
8107        Ok(())
8108    }
8109}
8110
8111#[derive(Debug)]
8112struct WorkflowState {
8113    workflow_id: Option<String>,
8114    run_id: Option<String>,
8115    task_queue: String,
8116    payload_codec: String,
8117    history_events: Arc<Vec<HistoryEvent>>,
8118    history_budget: WorkflowHistoryBudget,
8119    cancel_requested: bool,
8120    resume_signal: Option<ResumeSignal>,
8121    recorded_commands: Vec<RecordedCommand>,
8122    recorded_continue_as_new_sequence: Option<u64>,
8123    continue_as_new_consumed: bool,
8124    command_cursor: usize,
8125    matched_recorded_pending: bool,
8126    version_markers: HashMap<String, (i32, u64)>,
8127    workflow_command_identity: String,
8128    workflow_stream_command_counter: u64,
8129    commands: Vec<Value>,
8130}
8131
8132impl WorkflowState {
8133    #[cfg(test)]
8134    fn new(
8135        history: Vec<HistoryEvent>,
8136        task_queue: String,
8137        payload_codec: String,
8138        resume_signal: Option<ResumeSignal>,
8139    ) -> Result<Self> {
8140        Self::new_with_identity(
8141            history,
8142            None,
8143            None,
8144            task_queue,
8145            payload_codec,
8146            resume_signal,
8147        )
8148    }
8149
8150    fn new_with_identity(
8151        history: Vec<HistoryEvent>,
8152        workflow_id: Option<String>,
8153        run_id: Option<String>,
8154        task_queue: String,
8155        payload_codec: String,
8156        resume_signal: Option<ResumeSignal>,
8157    ) -> Result<Self> {
8158        let recorded_commands = recorded_commands(
8159            &history,
8160            &payload_codec,
8161            WorkflowIdentity {
8162                workflow_id: workflow_id.clone(),
8163                run_id: run_id.clone(),
8164            },
8165        )?;
8166        let recorded_continue_as_new = history
8167            .iter()
8168            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
8169            .collect::<Vec<_>>();
8170        if recorded_continue_as_new.len() > 1 {
8171            return Err(invalid_recorded_history(
8172                "duplicate_continue_as_new_transition",
8173                recorded_continue_as_new
8174                    .last()
8175                    .and_then(|event| durable_event_sequence(event))
8176                    .unwrap_or(0),
8177                "one WorkflowContinuedAsNew event",
8178                &format!(
8179                    "{} WorkflowContinuedAsNew events",
8180                    recorded_continue_as_new.len()
8181                ),
8182                "workflow history records one continue-as-new transition more than once",
8183            ));
8184        }
8185        let recorded_continue_as_new_sequence = recorded_continue_as_new
8186            .first()
8187            .map(|event| {
8188                durable_event_sequence(event).ok_or_else(|| {
8189                    Error::NonDeterministicReplay(ReplayFailure::new(
8190                        "continue_as_new_sequence_missing",
8191                        None,
8192                        Some("recorded transition sequence".to_string()),
8193                        Some("missing sequence".to_string()),
8194                        "WorkflowContinuedAsNew history is missing its recorded sequence",
8195                    ))
8196                })
8197            })
8198            .transpose()?;
8199        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
8200        let cancel_requested = history.iter().any(|event| {
8201            matches!(
8202                event.event_type.as_str(),
8203                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
8204            )
8205        });
8206        Ok(Self {
8207            workflow_command_identity: String::new(),
8208            workflow_stream_command_counter: 0,
8209            workflow_id,
8210            run_id,
8211            task_queue,
8212            payload_codec,
8213            history_events: Arc::new(history),
8214            history_budget: WorkflowHistoryBudget {
8215                event_count,
8216                ..WorkflowHistoryBudget::default()
8217            },
8218            cancel_requested,
8219            resume_signal,
8220            recorded_commands,
8221            recorded_continue_as_new_sequence,
8222            continue_as_new_consumed: false,
8223            command_cursor: 0,
8224            matched_recorded_pending: false,
8225            version_markers: HashMap::new(),
8226            commands: Vec::new(),
8227        })
8228    }
8229}
8230
8231#[derive(Clone, Debug)]
8232enum RecordedCommand {
8233    Activity {
8234        sequence: u64,
8235        activity_type: Option<String>,
8236        options: Option<RecordedActivityOptions>,
8237        outcome: Option<ActivityOutcome>,
8238        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8239    },
8240    Timer {
8241        sequence: u64,
8242        delay_seconds: u64,
8243        fired: bool,
8244        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8245    },
8246    ChildWorkflow {
8247        sequence: u64,
8248        workflow_type: Option<String>,
8249        outcome: Option<ChildWorkflowOutcome>,
8250        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
8251    },
8252    SignalWait {
8253        sequence: u64,
8254        signal_name: String,
8255        value: Option<Vec<AvroValue>>,
8256    },
8257    ConditionWait {
8258        sequence: u64,
8259        condition_key: Option<String>,
8260        predicate_identity: String,
8261        timeout_seconds: Option<u64>,
8262        result: Option<ConditionWaitResult>,
8263    },
8264    SearchAttributes {
8265        sequence: u64,
8266        attributes: Value,
8267        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
8268    },
8269    SideEffect {
8270        sequence: u64,
8271        value: AvroValue,
8272    },
8273    VersionMarker {
8274        sequence: u64,
8275        change_id: String,
8276        version: i32,
8277    },
8278    Memo {
8279        sequence: u64,
8280        entries: AvroValue,
8281    },
8282}
8283
8284#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8285struct RecordedActivityOptions {
8286    task_queue: RecordedSnapshotValue<Option<String>>,
8287    execution_mode: RecordedSnapshotValue<Option<String>>,
8288    retry_policy: ActivityRetrySnapshot,
8289}
8290
8291#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8292enum RecordedSnapshotValue<T> {
8293    /// Older history did not persist this field, so it cannot constrain replay.
8294    Unknown,
8295    Known(T),
8296}
8297
8298impl<T: PartialEq> RecordedSnapshotValue<T> {
8299    fn matches_current(&self, current: &Self) -> bool {
8300        match self {
8301            Self::Unknown => true,
8302            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
8303        }
8304    }
8305}
8306
8307#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
8308struct ActivityRetrySnapshot {
8309    snapshot_version: RecordedSnapshotValue<Option<u64>>,
8310    max_attempts: RecordedSnapshotValue<Option<u64>>,
8311    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
8312    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
8313    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
8314    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
8315    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
8316    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
8317}
8318
8319impl ActivityRetrySnapshot {
8320    fn matches_current(&self, current: &Self) -> bool {
8321        self.snapshot_version
8322            .matches_current(&current.snapshot_version)
8323            && self.max_attempts.matches_current(&current.max_attempts)
8324            && self
8325                .backoff_seconds
8326                .matches_current(&current.backoff_seconds)
8327            && self
8328                .start_to_close_timeout
8329                .matches_current(&current.start_to_close_timeout)
8330            && self
8331                .schedule_to_start_timeout
8332                .matches_current(&current.schedule_to_start_timeout)
8333            && self
8334                .schedule_to_close_timeout
8335                .matches_current(&current.schedule_to_close_timeout)
8336            && self
8337                .heartbeat_timeout
8338                .matches_current(&current.heartbeat_timeout)
8339            && self
8340                .non_retryable_error_types
8341                .matches_current(&current.non_retryable_error_types)
8342    }
8343}
8344
8345fn recorded_optional_u64(
8346    object: Option<&serde_json::Map<String, Value>>,
8347    field: &str,
8348) -> RecordedSnapshotValue<Option<u64>> {
8349    match object.and_then(|object| object.get(field)) {
8350        None => RecordedSnapshotValue::Unknown,
8351        Some(Value::Null) => RecordedSnapshotValue::Known(None),
8352        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
8353    }
8354}
8355
8356fn recorded_optional_string(
8357    object: &serde_json::Map<String, Value>,
8358    field: &str,
8359) -> RecordedSnapshotValue<Option<String>> {
8360    match object.get(field) {
8361        None => RecordedSnapshotValue::Unknown,
8362        Some(Value::Null) => RecordedSnapshotValue::Known(None),
8363        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
8364    }
8365}
8366
8367fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
8368    let policy = policy.and_then(Value::as_object);
8369    let backoff_seconds = policy
8370        .and_then(|policy| policy.get("backoff_seconds"))
8371        .and_then(Value::as_array)
8372        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
8373        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
8374    let mut non_retryable_error_types = Vec::new();
8375    for error_type in policy
8376        .and_then(|policy| policy.get("non_retryable_error_types"))
8377        .and_then(Value::as_array)
8378        .into_iter()
8379        .flatten()
8380        .filter_map(Value::as_str)
8381        .map(str::trim)
8382        .filter(|error_type| !error_type.is_empty())
8383    {
8384        if !non_retryable_error_types
8385            .iter()
8386            .any(|recorded| recorded == error_type)
8387        {
8388            non_retryable_error_types.push(error_type.to_string());
8389        }
8390    }
8391
8392    ActivityRetrySnapshot {
8393        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
8394        max_attempts: recorded_optional_u64(policy, "max_attempts"),
8395        backoff_seconds,
8396        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
8397        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
8398        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
8399        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
8400        non_retryable_error_types: if policy
8401            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
8402        {
8403            RecordedSnapshotValue::Known(non_retryable_error_types)
8404        } else {
8405            RecordedSnapshotValue::Unknown
8406        },
8407    }
8408}
8409
8410fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
8411    let policy = options.retry_policy.as_ref();
8412    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
8413        Some(Value::Null) => None,
8414        Some(value) => value_as_u64(value),
8415        None => Some(1),
8416    };
8417    let backoff_seconds = policy
8418        .and_then(|policy| policy.get("backoff_seconds"))
8419        .and_then(Value::as_array)
8420        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
8421        .unwrap_or_default();
8422    let non_retryable_error_types = policy
8423        .and_then(|policy| policy.get("non_retryable_error_types"))
8424        .and_then(Value::as_array)
8425        .into_iter()
8426        .flatten()
8427        .filter_map(Value::as_str)
8428        .map(str::to_string)
8429        .collect();
8430
8431    ActivityRetrySnapshot {
8432        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
8433        max_attempts: RecordedSnapshotValue::Known(max_attempts),
8434        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
8435        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
8436        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
8437        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
8438        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
8439        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
8440    }
8441}
8442
8443fn activity_options_description(options: &RecordedActivityOptions) -> String {
8444    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
8445}
8446
8447impl RecordedCommand {
8448    fn sequence(&self) -> u64 {
8449        match self {
8450            Self::Activity { sequence, .. }
8451            | Self::Timer { sequence, .. }
8452            | Self::ChildWorkflow { sequence, .. }
8453            | Self::SignalWait { sequence, .. }
8454            | Self::ConditionWait { sequence, .. }
8455            | Self::SearchAttributes { sequence, .. }
8456            | Self::SideEffect { sequence, .. }
8457            | Self::VersionMarker { sequence, .. }
8458            | Self::Memo { sequence, .. } => *sequence,
8459        }
8460    }
8461
8462    fn shape(&self) -> &'static str {
8463        match self {
8464            Self::Activity { .. } => "activity",
8465            Self::Timer { .. } => "timer",
8466            Self::ChildWorkflow { .. } => "child workflow",
8467            Self::SignalWait { .. } => "signal wait",
8468            Self::ConditionWait { .. } => "condition wait",
8469            Self::SearchAttributes { .. } => "search-attribute update",
8470            Self::SideEffect { .. } => "side effect",
8471            Self::VersionMarker { .. } => "version marker",
8472            Self::Memo { .. } => "memo upsert",
8473        }
8474    }
8475}
8476
8477fn ensure_version_supported(
8478    change_id: &str,
8479    version: i32,
8480    min_supported: i32,
8481    max_supported: i32,
8482    sequence: u64,
8483) -> Result<()> {
8484    if (min_supported..=max_supported).contains(&version) {
8485        return Ok(());
8486    }
8487    Err(Error::NonDeterministicReplay(ReplayFailure::new(
8488        "version_marker_incompatible_range",
8489        (sequence != 0).then_some(sequence),
8490        Some(format!("{min_supported}..={max_supported}")),
8491        Some(format!("{change_id}:{version}")),
8492        "recorded workflow version is outside the range supported by current code",
8493    )))
8494}
8495
8496#[derive(Clone, Debug)]
8497struct ResumeSignal {
8498    signal_name: String,
8499    arguments: Vec<AvroValue>,
8500}
8501
8502const MAX_PARALLEL_OPERATIONS: usize = 1000;
8503
8504fn parallel_group_prefix(kind: &str) -> &'static str {
8505    match kind {
8506        "activity" => "parallel-activities",
8507        "child" => "parallel-children",
8508        "timer" => "parallel-timers",
8509        _ => "parallel-calls",
8510    }
8511}
8512
8513fn parallel_group_entry(
8514    base_sequence: u64,
8515    size: usize,
8516    index: usize,
8517    kind: &str,
8518) -> ParallelGroupMetadata {
8519    ParallelGroupMetadata {
8520        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
8521        parallel_group_kind: kind.to_string(),
8522        parallel_group_base_sequence: base_sequence,
8523        parallel_group_size: size,
8524        parallel_group_index: index,
8525    }
8526}
8527
8528fn apply_parallel_group_path(
8529    command: &mut serde_json::Map<String, Value>,
8530    path: &[ParallelGroupMetadata],
8531) {
8532    let Some(inner) = path.last() else {
8533        return;
8534    };
8535    command.insert(
8536        "parallel_group_id".to_string(),
8537        json!(inner.parallel_group_id),
8538    );
8539    command.insert(
8540        "parallel_group_kind".to_string(),
8541        json!(inner.parallel_group_kind),
8542    );
8543    command.insert(
8544        "parallel_group_base_sequence".to_string(),
8545        json!(inner.parallel_group_base_sequence),
8546    );
8547    command.insert(
8548        "parallel_group_size".to_string(),
8549        json!(inner.parallel_group_size),
8550    );
8551    command.insert(
8552        "parallel_group_index".to_string(),
8553        json!(inner.parallel_group_index),
8554    );
8555    command.insert("parallel_group_path".to_string(), json!(path));
8556}
8557
8558fn ensure_parallel_path_matches(
8559    sequence: u64,
8560    recorded: Option<&[ParallelGroupMetadata]>,
8561    expected: &[ParallelGroupMetadata],
8562) -> Result<()> {
8563    match (recorded, expected.is_empty()) {
8564        (None, true) => Ok(()),
8565        (Some(recorded), false) if recorded == expected => Ok(()),
8566        (None, false) => Err(invalid_recorded_history(
8567            "parallel_group_metadata_missing",
8568            sequence,
8569            &serde_json::to_string(expected).unwrap_or_default(),
8570            "<missing>",
8571            "recorded parallel member is missing its durable group path",
8572        )),
8573        (Some(recorded), true) => Err(invalid_recorded_history(
8574            "parallel_group_shape_mismatch",
8575            sequence,
8576            "sequential command",
8577            &serde_json::to_string(recorded).unwrap_or_default(),
8578            "recorded command belonged to a parallel group but current code schedules it sequentially",
8579        )),
8580        (Some(recorded), false) => Err(invalid_recorded_history(
8581            "parallel_group_shape_mismatch",
8582            sequence,
8583            &serde_json::to_string(recorded).unwrap_or_default(),
8584            &serde_json::to_string(expected).unwrap_or_default(),
8585            "recorded parallel-group identity or path changed during replay",
8586        )),
8587    }
8588}
8589
8590#[derive(Clone, Debug)]
8591enum ParallelShape {
8592    Leaf,
8593    Group(Vec<ParallelShape>),
8594}
8595
8596struct ParallelDescriptor {
8597    operation: ParallelOperation,
8598    offset: usize,
8599    member_path: Vec<usize>,
8600    group_path: Vec<ParallelGroupMetadata>,
8601}
8602
8603fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
8604    operations
8605        .iter()
8606        .map(|operation| match operation {
8607            ParallelOperation::Group(children) => parallel_leaf_count(children),
8608            _ => 1,
8609        })
8610        .sum()
8611}
8612
8613fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
8614    match operation {
8615        ParallelOperation::Activity { .. } => Some("activity"),
8616        ParallelOperation::ChildWorkflow { .. } => Some("child"),
8617        ParallelOperation::Timer(_) => Some("timer"),
8618        ParallelOperation::Group(children) => parallel_group_kind(children),
8619    }
8620}
8621
8622fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
8623    let mut kind = None;
8624    for operation in operations {
8625        let Some(operation_kind) = parallel_operation_kind(operation) else {
8626            continue;
8627        };
8628        match kind {
8629            None => kind = Some(operation_kind),
8630            Some(current) if current == operation_kind => {}
8631            Some(_) => return Some("mixed"),
8632        }
8633    }
8634    kind
8635}
8636
8637fn validate_parallel_operations(
8638    operations: &[ParallelOperation],
8639    member_path: &mut Vec<usize>,
8640    root: bool,
8641) -> Result<()> {
8642    let leaves = parallel_leaf_count(operations);
8643    if leaves > MAX_PARALLEL_OPERATIONS {
8644        return Err(Error::InvalidParallelGroup(ParallelGroupError {
8645            reason: "fan_out_limit_exceeded",
8646            member_path: member_path.clone(),
8647            message: format!(
8648                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
8649            ),
8650        }));
8651    }
8652    if !root && operations.is_empty() {
8653        return Err(Error::InvalidParallelGroup(ParallelGroupError {
8654            reason: "nested_group_empty",
8655            member_path: member_path.clone(),
8656            message: "a nested group must contain at least one durable leaf".to_string(),
8657        }));
8658    }
8659
8660    for (index, operation) in operations.iter().enumerate() {
8661        member_path.push(index);
8662        match operation {
8663            ParallelOperation::Activity {
8664                options, arguments, ..
8665            } => {
8666                options
8667                    .validate()
8668                    .map_err(|error| Error::InvalidActivityOptions(error))?;
8669                if let Err(error) = arguments {
8670                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
8671                        reason: "arguments_invalid",
8672                        member_path: member_path.clone(),
8673                        message: error.to_string(),
8674                    }));
8675                }
8676            }
8677            ParallelOperation::ChildWorkflow {
8678                options, arguments, ..
8679            } => {
8680                validate_parallel_child_options(options)?;
8681                if let Err(error) = arguments {
8682                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
8683                        reason: "arguments_invalid",
8684                        member_path: member_path.clone(),
8685                        message: error.to_string(),
8686                    }));
8687                }
8688            }
8689            ParallelOperation::Timer(duration)
8690                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
8691            {
8692                return Err(Error::TimerDurationOverflow);
8693            }
8694            ParallelOperation::Timer(_) => {}
8695            ParallelOperation::Group(children) => {
8696                validate_parallel_operations(children, member_path, false)?;
8697            }
8698        }
8699        member_path.pop();
8700    }
8701    Ok(())
8702}
8703
8704fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
8705    if options.task_queue.trim().is_empty() {
8706        return Err(Error::InvalidChildWorkflowOptions(
8707            "task_queue must not be empty".to_string(),
8708        ));
8709    }
8710    for (name, value) in [
8711        (
8712            "execution_timeout_seconds",
8713            options.execution_timeout_seconds,
8714        ),
8715        ("run_timeout_seconds", options.run_timeout_seconds),
8716    ] {
8717        if value == Some(0) {
8718            return Err(Error::InvalidChildWorkflowOptions(format!(
8719                "{name} must be at least 1"
8720            )));
8721        }
8722    }
8723    if options
8724        .retry_policy
8725        .as_ref()
8726        .is_some_and(|policy| policy.max_attempts == Some(0))
8727    {
8728        return Err(Error::InvalidChildWorkflowOptions(
8729            "retry_policy.max_attempts must be at least 1".to_string(),
8730        ));
8731    }
8732    Ok(())
8733}
8734
8735fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
8736    ParallelShape::Group(
8737        operations
8738            .iter()
8739            .map(|operation| match operation {
8740                ParallelOperation::Group(children) => parallel_shape(children),
8741                _ => ParallelShape::Leaf,
8742            })
8743            .collect(),
8744    )
8745}
8746
8747fn parallel_descriptors(
8748    operations: Vec<ParallelOperation>,
8749    base_sequence: u64,
8750) -> Result<Vec<ParallelDescriptor>> {
8751    let size = parallel_leaf_count(&operations);
8752    let kind = parallel_group_kind(&operations).unwrap_or("activity");
8753    let mut descriptors = Vec::with_capacity(size);
8754    let mut cursor = 0;
8755
8756    for (index, operation) in operations.into_iter().enumerate() {
8757        match operation {
8758            ParallelOperation::Group(children) => {
8759                let child_base = base_sequence
8760                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
8761                    .ok_or(Error::TimerDurationOverflow)?;
8762                for mut descriptor in parallel_descriptors(children, child_base)? {
8763                    let outer_index = cursor + descriptor.offset;
8764                    descriptor.group_path.insert(
8765                        0,
8766                        parallel_group_entry(base_sequence, size, outer_index, kind),
8767                    );
8768                    descriptor.member_path.insert(0, index);
8769                    descriptor.offset = outer_index;
8770                    descriptors.push(descriptor);
8771                }
8772                cursor = descriptors.len();
8773            }
8774            operation => {
8775                descriptors.push(ParallelDescriptor {
8776                    operation,
8777                    offset: cursor,
8778                    member_path: vec![index],
8779                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
8780                });
8781                cursor += 1;
8782            }
8783        }
8784    }
8785    Ok(descriptors)
8786}
8787
8788enum ParallelLeafCall {
8789    Activity(ActivityCall),
8790    ChildWorkflow(ChildWorkflowCall),
8791    Timer(TimerCall),
8792}
8793
8794impl ParallelLeafCall {
8795    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
8796        match self {
8797            Self::Activity(call) => Pin::new(call)
8798                .poll_avro_value(cx)
8799                .map_ok(ParallelAvroResult::Activity),
8800            Self::ChildWorkflow(call) => Pin::new(call)
8801                .poll_avro_value(cx)
8802                .map_ok(ParallelAvroResult::ChildWorkflow),
8803            Self::Timer(call) => Pin::new(call)
8804                .poll(cx)
8805                .map_ok(|()| ParallelAvroResult::Timer),
8806        }
8807    }
8808}
8809
8810struct ParallelLeaf {
8811    call: ParallelLeafCall,
8812    member_path: Vec<usize>,
8813    group_path: Vec<ParallelGroupMetadata>,
8814    result: Option<ParallelAvroResult>,
8815}
8816
8817/// Future returned by [`WorkflowContext::parallel`].
8818pub struct ParallelCall {
8819    ctx: WorkflowContext,
8820    operations: Option<Vec<ParallelOperation>>,
8821    shape: Option<ParallelShape>,
8822    leaves: Vec<ParallelLeaf>,
8823}
8824
8825impl ParallelCall {
8826    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
8827        Self {
8828            ctx,
8829            operations: Some(operations),
8830            shape: None,
8831            leaves: Vec::new(),
8832        }
8833    }
8834
8835    fn initialize(&mut self) -> Result<()> {
8836        let operations = self.operations.take().unwrap_or_default();
8837        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
8838        self.shape = Some(parallel_shape(&operations));
8839        if operations.is_empty() {
8840            return Ok(());
8841        }
8842
8843        let base_sequence = {
8844            let state = self
8845                .ctx
8846                .state
8847                .lock()
8848                .map_err(|_| Error::WorkflowStatePoisoned)?;
8849            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
8850                recorded.sequence()
8851            } else {
8852                let last = state
8853                    .recorded_commands
8854                    .last()
8855                    .map(RecordedCommand::sequence)
8856                    .unwrap_or(0);
8857                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
8858                    .and_then(|sequence| sequence.checked_add(1))
8859                    .ok_or_else(|| {
8860                        Error::InvalidParallelGroup(ParallelGroupError {
8861                            reason: "sequence_overflow",
8862                            member_path: Vec::new(),
8863                            message: "parallel group sequence identity overflowed u64".to_string(),
8864                        })
8865                    })?
8866            }
8867        };
8868
8869        self.leaves = parallel_descriptors(operations, base_sequence)?
8870            .into_iter()
8871            .map(|descriptor| {
8872                let path = descriptor.group_path.clone();
8873                let call = match descriptor.operation {
8874                    ParallelOperation::Activity {
8875                        activity_type,
8876                        options,
8877                        arguments,
8878                    } => ParallelLeafCall::Activity(ActivityCall {
8879                        ctx: self.ctx.clone(),
8880                        activity_type,
8881                        options,
8882                        args: Some(arguments),
8883                        scheduled: false,
8884                        parallel_group_path: path,
8885                    }),
8886                    ParallelOperation::ChildWorkflow {
8887                        workflow_type,
8888                        options,
8889                        arguments,
8890                    } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
8891                        ctx: self.ctx.clone(),
8892                        workflow_type,
8893                        options,
8894                        args: Some(arguments),
8895                        scheduled: false,
8896                        matched_pending: false,
8897                        parallel_group_path: path,
8898                    }),
8899                    ParallelOperation::Timer(duration) => {
8900                        let delay_seconds = duration
8901                            .as_secs()
8902                            .checked_add(u64::from(duration.subsec_nanos() > 0));
8903                        ParallelLeafCall::Timer(TimerCall {
8904                            ctx: self.ctx.clone(),
8905                            delay_seconds,
8906                            scheduled: false,
8907                            matched_pending: false,
8908                            parallel_group_path: path,
8909                        })
8910                    }
8911                    ParallelOperation::Group(_) => {
8912                        unreachable!("parallel descriptors contain only durable leaves")
8913                    }
8914                };
8915                ParallelLeaf {
8916                    call,
8917                    member_path: descriptor.member_path,
8918                    group_path: descriptor.group_path,
8919                    result: None,
8920                }
8921            })
8922            .collect();
8923        Ok(())
8924    }
8925
8926    fn poll_avro_value(
8927        mut self: Pin<&mut Self>,
8928        cx: &mut TaskContext<'_>,
8929    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
8930        if self.operations.is_some() {
8931            if let Err(error) = self.initialize() {
8932                return Poll::Ready(Err(error));
8933            }
8934        }
8935        if self.leaves.is_empty() {
8936            return Poll::Ready(Ok(Vec::new()));
8937        }
8938
8939        let mut failures = Vec::new();
8940        let mut pending = false;
8941        for (index, leaf) in self.leaves.iter_mut().enumerate() {
8942            if leaf.result.is_some() {
8943                continue;
8944            }
8945            match leaf.call.poll_avro_value(cx) {
8946                Poll::Ready(Ok(result)) => leaf.result = Some(result),
8947                Poll::Ready(Err(error)) => failures.push((index, error)),
8948                Poll::Pending => pending = true,
8949            }
8950        }
8951
8952        if !failures.is_empty() {
8953            if let Some(position) = failures
8954                .iter()
8955                .position(|(_, error)| workflow_task_integrity_error(error))
8956            {
8957                return Poll::Ready(Err(failures.remove(position).1));
8958            }
8959            failures.sort_by_key(|(index, _)| *index);
8960            let (failed_index, cause) = failures.remove(0);
8961            let failed = &self.leaves[failed_index];
8962            let completed = self
8963                .leaves
8964                .iter()
8965                .filter_map(|leaf| {
8966                    leaf.result
8967                        .clone()
8968                        .and_then(|result| result.into_json_result().ok())
8969                        .map(|result| ParallelCompletion {
8970                            member_path: leaf.member_path.clone(),
8971                            result,
8972                        })
8973                })
8974                .collect();
8975            let group_id = failed
8976                .group_path
8977                .first()
8978                .map(|entry| entry.parallel_group_id.clone())
8979                .unwrap_or_default();
8980            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
8981                group_id,
8982                member_path: failed.member_path.clone(),
8983                group_path: failed.group_path.clone(),
8984                completed,
8985                cause: Box::new(cause),
8986            })));
8987        }
8988        if pending {
8989            return Poll::Pending;
8990        }
8991
8992        let mut flat_results = self
8993            .leaves
8994            .iter_mut()
8995            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
8996            .collect::<Vec<_>>()
8997            .into_iter();
8998        let results = parallel_results_for_shape(
8999            self.shape.as_ref().expect("initialized parallel shape"),
9000            &mut flat_results,
9001        );
9002        Poll::Ready(Ok(match results {
9003            ParallelAvroResult::Group(results) => results,
9004            ParallelAvroResult::Activity(_)
9005            | ParallelAvroResult::ChildWorkflow(_)
9006            | ParallelAvroResult::Timer => unreachable!("root parallel shape is a group"),
9007        }))
9008    }
9009}
9010
9011fn parallel_results_for_shape(
9012    shape: &ParallelShape,
9013    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
9014) -> ParallelAvroResult {
9015    match shape {
9016        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
9017        ParallelShape::Group(children) => ParallelAvroResult::Group(
9018            children
9019                .iter()
9020                .map(|child| parallel_results_for_shape(child, flat_results))
9021                .collect(),
9022        ),
9023    }
9024}
9025
9026impl Future for ParallelCall {
9027    type Output = Result<Vec<ParallelResult>>;
9028
9029    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9030        self.poll_avro_value(cx)
9031            .map_ok(|results| {
9032                results
9033                    .into_iter()
9034                    .map(ParallelAvroResult::into_json_result)
9035                    .collect::<Result<Vec<_>>>()
9036            })
9037            .map_ok(|result| result)
9038            .flatten_result()
9039    }
9040}
9041
9042trait PollNestedResultExt<T> {
9043    fn flatten_result(self) -> Poll<Result<T>>;
9044}
9045
9046impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
9047    fn flatten_result(self) -> Poll<Result<T>> {
9048        match self {
9049            Poll::Ready(Ok(result)) => Poll::Ready(result),
9050            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9051            Poll::Pending => Poll::Pending,
9052        }
9053    }
9054}
9055
9056struct SagaCompensation {
9057    activity_type: String,
9058    options: ActivityOptions,
9059    arguments: AvroValue,
9060    registration_order: usize,
9061}
9062
9063/// Workflow-local deterministic saga compensation helper.
9064///
9065/// Register each compensation only after its forward step succeeds. Passing
9066/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
9067/// reverse registration order after any failure, including cooperative
9068/// cancellation. Each compensation is an ordinary durable activity, so replay,
9069/// duplicate delivery, and worker restart use existing history semantics.
9070pub struct Saga {
9071    ctx: WorkflowContext,
9072    compensations: Vec<SagaCompensation>,
9073}
9074
9075impl Saga {
9076    fn new(ctx: WorkflowContext) -> Self {
9077        Self {
9078            ctx,
9079            compensations: Vec::new(),
9080        }
9081    }
9082
9083    pub fn add_compensation<T: Serialize>(
9084        &mut self,
9085        activity_type: impl Into<String>,
9086        args: T,
9087    ) -> Result<&mut Self> {
9088        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
9089    }
9090
9091    pub fn add_compensation_with_options<T: Serialize>(
9092        &mut self,
9093        activity_type: impl Into<String>,
9094        options: ActivityOptions,
9095        args: T,
9096    ) -> Result<&mut Self> {
9097        let activity_type = activity_type.into();
9098        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
9099            return Err(Error::Codec(
9100                "saga compensation activity type must be non-empty without surrounding whitespace"
9101                    .to_string(),
9102            ));
9103        }
9104        options.validate().map_err(Error::InvalidActivityOptions)?;
9105        let arguments = AvroValue::from_serialize(&args)?;
9106        let registration_order = self.compensations.len() + 1;
9107        self.compensations.push(SagaCompensation {
9108            activity_type,
9109            options,
9110            arguments,
9111            registration_order,
9112        });
9113        Ok(self)
9114    }
9115
9116    /// Compensate `initiating_failure` and return the failure that must remain.
9117    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
9118        while let Some(compensation) = self.compensations.pop() {
9119            if let Err(compensation_failure) = self
9120                .ctx
9121                .activity_with_options(
9122                    compensation.activity_type.clone(),
9123                    compensation.options,
9124                    compensation.arguments,
9125                )
9126                .await
9127            {
9128                if workflow_task_integrity_error(&compensation_failure) {
9129                    return compensation_failure;
9130                }
9131                return Error::SagaCompensationFailed(SagaCompensationFailure {
9132                    initiating_failure: Box::new(initiating_failure),
9133                    compensation_failure: Box::new(compensation_failure),
9134                    compensation_activity_type: compensation.activity_type,
9135                    compensation_registration_order: compensation.registration_order,
9136                });
9137            }
9138        }
9139        initiating_failure
9140    }
9141
9142    /// Return a successful forward value or compensate and preserve its failure.
9143    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
9144        match outcome {
9145            Ok(value) => Ok(value),
9146            Err(error) => Err(self.compensate(error).await),
9147        }
9148    }
9149}
9150
9151pub struct ActivityCall {
9152    ctx: WorkflowContext,
9153    activity_type: String,
9154    options: ActivityOptions,
9155    args: Option<Result<AvroValue>>,
9156    scheduled: bool,
9157    parallel_group_path: Vec<ParallelGroupMetadata>,
9158}
9159
9160impl ActivityCall {
9161    fn poll_avro_value(
9162        mut self: Pin<&mut Self>,
9163        _cx: &mut TaskContext<'_>,
9164    ) -> Poll<Result<AvroValue>> {
9165        let ctx = self.ctx.clone();
9166        let mut state = match ctx.state.lock() {
9167            Ok(state) => state,
9168            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9169        };
9170
9171        if self.scheduled {
9172            return Poll::Pending;
9173        }
9174
9175        let options = match self.options.validate() {
9176            Ok(options) => options,
9177            Err(error) => {
9178                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
9179            }
9180        };
9181        let task_queue = options
9182            .task_queue
9183            .clone()
9184            .unwrap_or_else(|| state.task_queue.clone());
9185        let current_recorded_options = RecordedActivityOptions {
9186            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
9187            // Rust schedules ordinary durable activities. The server records a
9188            // non-null mode only for a specialized execution primitive.
9189            execution_mode: RecordedSnapshotValue::Known(None),
9190            retry_policy: current_activity_retry_snapshot(&options),
9191        };
9192
9193        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9194            let sequence = recorded.sequence();
9195            match recorded {
9196                RecordedCommand::Activity {
9197                    activity_type,
9198                    options: recorded_options,
9199                    outcome,
9200                    parallel_group_path,
9201                    ..
9202                } => {
9203                    if let Err(error) = ensure_parallel_path_matches(
9204                        sequence,
9205                        parallel_group_path.as_deref(),
9206                        &self.parallel_group_path,
9207                    ) {
9208                        return Poll::Ready(Err(error));
9209                    }
9210                    if let Some(recorded_type) = activity_type {
9211                        if recorded_type != self.activity_type {
9212                            return Poll::Ready(Err(Error::NonDeterministicReplay(
9213                                ReplayFailure::new(
9214                                    "recorded_command_detail_mismatch",
9215                                    Some(sequence),
9216                                    Some(format!("activity:{recorded_type}")),
9217                                    Some(format!("activity:{}", self.activity_type)),
9218                                    "recorded activity type differs from the current workflow command",
9219                                ),
9220                            )));
9221                        }
9222                    }
9223                    if let Some(recorded_options) = recorded_options {
9224                        if !recorded_options
9225                            .task_queue
9226                            .matches_current(&current_recorded_options.task_queue)
9227                        {
9228                            return Poll::Ready(Err(Error::NonDeterministicReplay(
9229                                ReplayFailure::new(
9230                                    "activity_task_queue_mismatch",
9231                                    Some(sequence),
9232                                    Some(activity_options_description(&recorded_options)),
9233                                    Some(activity_options_description(&current_recorded_options)),
9234                                    "recorded activity task queue differs from the current workflow command",
9235                                ),
9236                            )));
9237                        }
9238                        if !recorded_options
9239                            .execution_mode
9240                            .matches_current(&current_recorded_options.execution_mode)
9241                        {
9242                            return Poll::Ready(Err(Error::NonDeterministicReplay(
9243                                ReplayFailure::new(
9244                                    "activity_execution_mode_mismatch",
9245                                    Some(sequence),
9246                                    Some(activity_options_description(&recorded_options)),
9247                                    Some(activity_options_description(&current_recorded_options)),
9248                                    "recorded activity execution mode differs from the current workflow command",
9249                                ),
9250                            )));
9251                        }
9252                        if !recorded_options
9253                            .retry_policy
9254                            .matches_current(&current_recorded_options.retry_policy)
9255                        {
9256                            return Poll::Ready(Err(Error::NonDeterministicReplay(
9257                                ReplayFailure::new(
9258                                    "activity_retry_policy_mismatch",
9259                                    Some(sequence),
9260                                    Some(activity_options_description(&recorded_options)),
9261                                    Some(activity_options_description(&current_recorded_options)),
9262                                    "recorded activity retry policy differs from the current workflow command",
9263                                ),
9264                            )));
9265                        }
9266                    }
9267                    state.command_cursor += 1;
9268                    if let Some(outcome) = outcome {
9269                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
9270                    }
9271                    state.matched_recorded_pending = true;
9272                    self.scheduled = true;
9273                    return Poll::Pending;
9274                }
9275                other => {
9276                    return Poll::Ready(Err(command_mismatch(
9277                        &other,
9278                        format!("activity:{}", self.activity_type),
9279                    )));
9280                }
9281            }
9282        }
9283
9284        if !self.scheduled {
9285            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
9286                Ok(args) => args,
9287                Err(error) => return Poll::Ready(Err(error)),
9288            };
9289            let arguments = normalize_avro_arguments(args);
9290            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
9291                Ok(envelope) => envelope,
9292                Err(error) => return Poll::Ready(Err(error)),
9293            };
9294
9295            let mut command = serde_json::Map::from_iter([
9296                ("type".to_string(), json!("schedule_activity")),
9297                (
9298                    "activity_type".to_string(),
9299                    json!(self.activity_type.clone()),
9300                ),
9301                ("queue".to_string(), json!(task_queue)),
9302                ("arguments".to_string(), envelope),
9303            ]);
9304            for (field, value) in [
9305                ("start_to_close_timeout", options.start_to_close_timeout),
9306                (
9307                    "schedule_to_start_timeout",
9308                    options.schedule_to_start_timeout,
9309                ),
9310                (
9311                    "schedule_to_close_timeout",
9312                    options.schedule_to_close_timeout,
9313                ),
9314                ("heartbeat_timeout", options.heartbeat_timeout),
9315            ] {
9316                if let Some(value) = value {
9317                    command.insert(field.to_string(), json!(value));
9318                }
9319            }
9320            if let Some(retry_policy) = options.retry_policy {
9321                command.insert("retry_policy".to_string(), retry_policy);
9322            }
9323            apply_parallel_group_path(&mut command, &self.parallel_group_path);
9324            state.commands.push(Value::Object(command));
9325            self.scheduled = true;
9326        }
9327
9328        Poll::Pending
9329    }
9330}
9331
9332impl Future for ActivityCall {
9333    type Output = Result<Value>;
9334
9335    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9336        match self.poll_avro_value(cx) {
9337            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
9338            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9339            Poll::Pending => Poll::Pending,
9340        }
9341    }
9342}
9343
9344/// Future returned by [`WorkflowContext::sleep`].
9345pub struct TimerCall {
9346    ctx: WorkflowContext,
9347    delay_seconds: Option<u64>,
9348    scheduled: bool,
9349    matched_pending: bool,
9350    parallel_group_path: Vec<ParallelGroupMetadata>,
9351}
9352
9353impl Future for TimerCall {
9354    type Output = Result<()>;
9355
9356    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9357        if self.matched_pending {
9358            return Poll::Pending;
9359        }
9360
9361        let ctx = self.ctx.clone();
9362        let Some(requested_delay) = self.delay_seconds else {
9363            return Poll::Ready(Err(Error::TimerDurationOverflow));
9364        };
9365        let mut state = match ctx.state.lock() {
9366            Ok(state) => state,
9367            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9368        };
9369
9370        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9371            match recorded {
9372                RecordedCommand::Timer {
9373                    sequence,
9374                    delay_seconds,
9375                    fired,
9376                    parallel_group_path,
9377                    ..
9378                } => {
9379                    if let Err(error) = ensure_parallel_path_matches(
9380                        sequence,
9381                        parallel_group_path.as_deref(),
9382                        &self.parallel_group_path,
9383                    ) {
9384                        return Poll::Ready(Err(error));
9385                    }
9386                    if delay_seconds != requested_delay {
9387                        return Poll::Ready(Err(Error::NonDeterministicReplay(
9388                            ReplayFailure::new(
9389                                "timer_delay_mismatch",
9390                                Some(sequence),
9391                                Some(format!("timer:{delay_seconds}s")),
9392                                Some(format!("timer:{requested_delay}s")),
9393                                "recorded timer delay differs from the current workflow command",
9394                            ),
9395                        )));
9396                    }
9397                    state.command_cursor += 1;
9398                    if fired {
9399                        return Poll::Ready(Ok(()));
9400                    }
9401                    state.matched_recorded_pending = true;
9402                    self.scheduled = true;
9403                    self.matched_pending = true;
9404                    return Poll::Pending;
9405                }
9406                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
9407            }
9408        }
9409
9410        if !self.scheduled {
9411            let mut command = serde_json::Map::from_iter([
9412                ("type".to_string(), json!("start_timer")),
9413                ("delay_seconds".to_string(), json!(requested_delay)),
9414            ]);
9415            apply_parallel_group_path(&mut command, &self.parallel_group_path);
9416            state.commands.push(Value::Object(command));
9417            self.scheduled = true;
9418        }
9419
9420        Poll::Pending
9421    }
9422}
9423
9424/// Future returned by [`WorkflowContext::wait_condition`].
9425pub struct ConditionWaitCall {
9426    ctx: WorkflowContext,
9427    options: ConditionWaitOptions,
9428    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
9429    opened_wait: bool,
9430}
9431
9432impl Future for ConditionWaitCall {
9433    type Output = Result<ConditionWaitResult>;
9434
9435    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9436        if self.opened_wait {
9437            return Poll::Pending;
9438        }
9439
9440        let options = match self.options.validate() {
9441            Ok(options) => options,
9442            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
9443        };
9444        let ctx = self.ctx.clone();
9445
9446        let recorded_result = {
9447            let mut state = match ctx.state.lock() {
9448                Ok(state) => state,
9449                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9450            };
9451            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
9452                drop(state);
9453                return self.poll_new_condition(options);
9454            };
9455            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
9456                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
9457            }
9458
9459            let mut cursor = state.command_cursor;
9460            let mut result = None;
9461            loop {
9462                let Some(RecordedCommand::ConditionWait {
9463                    sequence,
9464                    condition_key,
9465                    predicate_identity,
9466                    timeout_seconds,
9467                    result: recorded_result,
9468                    ..
9469                }) = state.recorded_commands.get(cursor)
9470                else {
9471                    break;
9472                };
9473
9474                if cursor > state.command_cursor
9475                    && !same_logical_condition_wait(
9476                        condition_key.as_deref(),
9477                        predicate_identity,
9478                        &options,
9479                    )
9480                {
9481                    break;
9482                }
9483                if let Err(error) = validate_recorded_condition_wait(
9484                    *sequence,
9485                    condition_key.as_deref(),
9486                    predicate_identity,
9487                    *timeout_seconds,
9488                    &options,
9489                ) {
9490                    return Poll::Ready(Err(error));
9491                }
9492                if result == Some(ConditionWaitResult::TimedOut) {
9493                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
9494                        "condition_wait_reopened_after_timeout",
9495                        Some(*sequence),
9496                        Some("timed-out condition is terminal".to_string()),
9497                        Some("another physical wait-open".to_string()),
9498                        "condition history reopened one logical wait after its durable timeout",
9499                    ))));
9500                }
9501                result = *recorded_result;
9502                cursor += 1;
9503            }
9504            state.command_cursor = cursor;
9505            result
9506        };
9507
9508        if let Some(result) = recorded_result {
9509            return Poll::Ready(Ok(result));
9510        }
9511
9512        self.poll_open_condition(options)
9513    }
9514}
9515
9516impl ConditionWaitCall {
9517    fn poll_new_condition(
9518        self: Pin<&mut Self>,
9519        options: ValidatedConditionWaitOptions,
9520    ) -> Poll<Result<ConditionWaitResult>> {
9521        self.poll_open_condition(options)
9522    }
9523
9524    fn poll_open_condition(
9525        mut self: Pin<&mut Self>,
9526        options: ValidatedConditionWaitOptions,
9527    ) -> Poll<Result<ConditionWaitResult>> {
9528        match (self.predicate)() {
9529            Ok(true) => return Poll::Ready(Ok(ConditionWaitResult::Satisfied)),
9530            Ok(false) => {}
9531            Err(error) => return Poll::Ready(Err(error)),
9532        }
9533        if options.timeout_seconds == Some(0) {
9534            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
9535        }
9536
9537        let ctx = self.ctx.clone();
9538        let mut state = match ctx.state.lock() {
9539            Ok(state) => state,
9540            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9541        };
9542        let mut command = serde_json::Map::from_iter([
9543            ("type".to_string(), json!("open_condition_wait")),
9544            ("condition_key".to_string(), json!(options.condition_key)),
9545            (
9546                "condition_definition_fingerprint".to_string(),
9547                json!(options.predicate_identity),
9548            ),
9549        ]);
9550        if let Some(timeout_seconds) = options.timeout_seconds {
9551            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
9552        }
9553        state.commands.push(Value::Object(command));
9554        drop(state);
9555        self.opened_wait = true;
9556        Poll::Pending
9557    }
9558}
9559
9560fn same_logical_condition_wait(
9561    recorded_key: Option<&str>,
9562    recorded_predicate_identity: &str,
9563    current: &ValidatedConditionWaitOptions,
9564) -> bool {
9565    recorded_key == Some(current.condition_key.as_str())
9566        || recorded_predicate_identity == current.predicate_identity
9567}
9568
9569fn validate_recorded_condition_wait(
9570    sequence: u64,
9571    recorded_key: Option<&str>,
9572    recorded_predicate_identity: &str,
9573    recorded_timeout_seconds: Option<u64>,
9574    current: &ValidatedConditionWaitOptions,
9575) -> Result<()> {
9576    if recorded_key != Some(current.condition_key.as_str()) {
9577        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9578            "condition_wait_key_mismatch",
9579            Some(sequence),
9580            recorded_key.map(str::to_string),
9581            Some(current.condition_key.clone()),
9582            "recorded condition identity differs from the current workflow wait",
9583        )));
9584    }
9585    if recorded_predicate_identity != current.predicate_identity {
9586        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9587            "condition_wait_predicate_mismatch",
9588            Some(sequence),
9589            Some(recorded_predicate_identity.to_string()),
9590            Some(current.predicate_identity.clone()),
9591            "recorded condition predicate behavior differs from current workflow code",
9592        )));
9593    }
9594    if recorded_timeout_seconds != current.timeout_seconds {
9595        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
9596            "condition_wait_timeout_mismatch",
9597            Some(sequence),
9598            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
9599            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
9600            "recorded condition timeout differs from the current workflow wait",
9601        )));
9602    }
9603    Ok(())
9604}
9605
9606/// Future returned by [`WorkflowContext::start_child_workflow`].
9607pub struct ChildWorkflowCall {
9608    ctx: WorkflowContext,
9609    workflow_type: String,
9610    options: ChildWorkflowOptions,
9611    args: Option<Result<AvroValue>>,
9612    scheduled: bool,
9613    matched_pending: bool,
9614    parallel_group_path: Vec<ParallelGroupMetadata>,
9615}
9616
9617impl ChildWorkflowCall {
9618    fn poll_avro_value(
9619        mut self: Pin<&mut Self>,
9620        _cx: &mut TaskContext<'_>,
9621    ) -> Poll<Result<ChildWorkflowAvroResult>> {
9622        if self.matched_pending {
9623            return Poll::Pending;
9624        }
9625
9626        let ctx = self.ctx.clone();
9627        let mut state = match ctx.state.lock() {
9628            Ok(state) => state,
9629            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9630        };
9631
9632        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9633            let sequence = recorded.sequence();
9634            match recorded {
9635                RecordedCommand::ChildWorkflow {
9636                    workflow_type,
9637                    outcome,
9638                    parallel_group_path,
9639                    ..
9640                } => {
9641                    if let Err(error) = ensure_parallel_path_matches(
9642                        sequence,
9643                        parallel_group_path.as_deref(),
9644                        &self.parallel_group_path,
9645                    ) {
9646                        return Poll::Ready(Err(error));
9647                    }
9648                    if let Some(recorded_type) = workflow_type {
9649                        if recorded_type != self.workflow_type {
9650                            return Poll::Ready(Err(Error::NonDeterministicReplay(
9651                                ReplayFailure::new(
9652                                    "recorded_command_detail_mismatch",
9653                                    Some(sequence),
9654                                    Some(format!("child workflow:{recorded_type}")),
9655                                    Some(format!("child workflow:{}", self.workflow_type)),
9656                                    "recorded child workflow type differs from the current workflow command",
9657                                ),
9658                            )));
9659                        }
9660                    }
9661                    state.command_cursor += 1;
9662                    if let Some(outcome) = outcome {
9663                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
9664                    }
9665                    state.matched_recorded_pending = true;
9666                    self.scheduled = true;
9667                    self.matched_pending = true;
9668                    return Poll::Pending;
9669                }
9670                other => {
9671                    return Poll::Ready(Err(command_mismatch(
9672                        &other,
9673                        format!("child workflow:{}", self.workflow_type),
9674                    )));
9675                }
9676            }
9677        }
9678
9679        if !self.scheduled {
9680            if self.options.task_queue.trim().is_empty() {
9681                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
9682                    "task_queue must not be empty".to_string(),
9683                )));
9684            }
9685            for (name, value) in [
9686                (
9687                    "execution_timeout_seconds",
9688                    self.options.execution_timeout_seconds,
9689                ),
9690                ("run_timeout_seconds", self.options.run_timeout_seconds),
9691            ] {
9692                if value == Some(0) {
9693                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
9694                        "{name} must be at least 1"
9695                    ))));
9696                }
9697            }
9698
9699            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
9700                Ok(args) => args,
9701                Err(error) => return Poll::Ready(Err(error)),
9702            };
9703            let arguments = match encode_typed_envelope(
9704                &normalize_avro_arguments(args),
9705                &state.payload_codec,
9706            ) {
9707                Ok(arguments) => arguments,
9708                Err(error) => return Poll::Ready(Err(error)),
9709            };
9710            let mut command = json!({
9711                "type": "start_child_workflow",
9712                "workflow_type": self.workflow_type,
9713                "queue": self.options.task_queue,
9714                "parent_close_policy": self.options.parent_close_policy.as_str(),
9715                "arguments": arguments,
9716            });
9717            let object = command
9718                .as_object_mut()
9719                .expect("child workflow command is always an object");
9720            if let Some(policy) = &self.options.retry_policy {
9721                let mut retry_policy = serde_json::Map::new();
9722                if let Some(max_attempts) = policy.max_attempts {
9723                    if max_attempts == 0 {
9724                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
9725                            "retry_policy.max_attempts must be at least 1".to_string(),
9726                        )));
9727                    }
9728                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
9729                }
9730                if !policy.backoff_seconds.is_empty() {
9731                    retry_policy
9732                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
9733                }
9734                if !policy.non_retryable_error_types.is_empty() {
9735                    retry_policy.insert(
9736                        "non_retryable_error_types".to_string(),
9737                        json!(policy.non_retryable_error_types),
9738                    );
9739                }
9740                if retry_policy.is_empty() {
9741                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
9742                        "retry_policy must configure at least one field".to_string(),
9743                    )));
9744                }
9745                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
9746            }
9747            if let Some(seconds) = self.options.execution_timeout_seconds {
9748                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
9749            }
9750            if let Some(seconds) = self.options.run_timeout_seconds {
9751                object.insert("run_timeout_seconds".to_string(), json!(seconds));
9752            }
9753            apply_parallel_group_path(object, &self.parallel_group_path);
9754            state.commands.push(command);
9755            self.scheduled = true;
9756        }
9757
9758        Poll::Pending
9759    }
9760}
9761
9762impl Future for ChildWorkflowCall {
9763    type Output = Result<ChildWorkflowResult>;
9764
9765    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9766        match self.poll_avro_value(cx) {
9767            Poll::Ready(Ok(result)) => match result.result.into_json() {
9768                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
9769                    parent: result.parent,
9770                    child: result.child,
9771                    child_workflow_type: result.child_workflow_type,
9772                    result: projected,
9773                })),
9774                Err(error) => Poll::Ready(Err(error)),
9775            },
9776            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9777            Poll::Pending => Poll::Pending,
9778        }
9779    }
9780}
9781
9782fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
9783    Error::NonDeterministicReplay(ReplayFailure::new(
9784        "recorded_command_mismatch",
9785        Some(recorded.sequence()),
9786        Some(recorded.shape().to_string()),
9787        Some(actual.into()),
9788        "current workflow command does not match the recorded durable command sequence",
9789    ))
9790}
9791
9792pub struct SignalCall {
9793    ctx: WorkflowContext,
9794    signal_name: String,
9795    opened_wait: bool,
9796    matched_pending: bool,
9797}
9798
9799impl SignalCall {
9800    fn poll_avro_value(
9801        mut self: Pin<&mut Self>,
9802        _cx: &mut TaskContext<'_>,
9803    ) -> Poll<Result<Vec<AvroValue>>> {
9804        if self.matched_pending {
9805            return Poll::Pending;
9806        }
9807
9808        let ctx = self.ctx.clone();
9809        let mut state = match ctx.state.lock() {
9810            Ok(state) => state,
9811            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
9812        };
9813
9814        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
9815            match recorded {
9816                RecordedCommand::SignalWait {
9817                    sequence,
9818                    signal_name,
9819                    value,
9820                } => {
9821                    if signal_name != self.signal_name {
9822                        return Poll::Ready(Err(Error::NonDeterministicReplay(
9823                            ReplayFailure::new(
9824                                "recorded_command_detail_mismatch",
9825                                Some(sequence),
9826                                Some(format!("signal wait:{signal_name}")),
9827                                Some(format!("signal wait:{}", self.signal_name)),
9828                                "recorded signal name differs from the current workflow command",
9829                            ),
9830                        )));
9831                    }
9832
9833                    state.command_cursor += 1;
9834                    if let Some(value) = value {
9835                        return Poll::Ready(Ok(value));
9836                    }
9837                    if state
9838                        .resume_signal
9839                        .as_ref()
9840                        .is_some_and(|signal| signal.signal_name == self.signal_name)
9841                    {
9842                        let signal = state
9843                            .resume_signal
9844                            .take()
9845                            .expect("matching resume signal is present");
9846                        return Poll::Ready(Ok(signal.arguments));
9847                    }
9848
9849                    state.matched_recorded_pending = true;
9850                    self.opened_wait = true;
9851                    self.matched_pending = true;
9852                    return Poll::Pending;
9853                }
9854                other => {
9855                    return Poll::Ready(Err(command_mismatch(
9856                        &other,
9857                        format!("signal wait:{}", self.signal_name),
9858                    )));
9859                }
9860            }
9861        }
9862
9863        if state
9864            .resume_signal
9865            .as_ref()
9866            .is_some_and(|signal| signal.signal_name == self.signal_name)
9867        {
9868            let signal = state
9869                .resume_signal
9870                .take()
9871                .expect("matching resume signal is present");
9872            return Poll::Ready(Ok(signal.arguments));
9873        }
9874
9875        if !self.opened_wait {
9876            state.commands.push(json!({
9877                "type": "open_signal_wait",
9878                "signal_name": self.signal_name
9879            }));
9880            self.opened_wait = true;
9881        }
9882
9883        Poll::Pending
9884    }
9885}
9886
9887impl Future for SignalCall {
9888    type Output = Result<Vec<Value>>;
9889
9890    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
9891        match self.poll_avro_value(cx) {
9892            Poll::Ready(Ok(values)) => Poll::Ready(
9893                values
9894                    .into_iter()
9895                    .map(AvroValue::into_json)
9896                    .collect::<Result<Vec<_>>>(),
9897            ),
9898            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
9899            Poll::Pending => Poll::Pending,
9900        }
9901    }
9902}
9903
9904#[derive(Clone, Debug)]
9905pub struct ActivityContext {
9906    client: Client,
9907    pub task_id: String,
9908    pub activity_attempt_id: String,
9909    pub lease_owner: String,
9910    pub activity_type: String,
9911    pub attempt_number: u64,
9912    pub task_queue: String,
9913    pub worker_id: String,
9914}
9915
9916impl ActivityContext {
9917    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
9918        self.client
9919            .heartbeat_activity_task(
9920                &self.task_id,
9921                &self.activity_attempt_id,
9922                &self.lease_owner,
9923                details,
9924            )
9925            .await
9926    }
9927}
9928
9929fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
9930    validate_payload_codec(codec)?;
9931    match value {
9932        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
9933            value, codec,
9934        )?)),
9935        None => Ok(AvroValue::Array(Vec::new())),
9936    }
9937}
9938
9939fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
9940    let Some(signal_name) = task
9941        .signal_name
9942        .as_deref()
9943        .filter(|value| !value.is_empty())
9944    else {
9945        return Ok(None);
9946    };
9947    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
9948    let AvroValue::Array(arguments) = decoded else {
9949        unreachable!("normalize_avro_arguments always returns an array");
9950    };
9951
9952    Ok(Some(ResumeSignal {
9953        signal_name: signal_name.to_string(),
9954        arguments,
9955    }))
9956}
9957
9958fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
9959    validate_payload_codec(&task.payload_codec)?;
9960    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
9961    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
9962    for event in &task.history_events {
9963        validate_history_event_payloads(event, &task.payload_codec)?;
9964    }
9965    Ok(())
9966}
9967
9968fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
9969    validate_payload_codec(&task.payload_codec)?;
9970    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
9971}
9972
9973fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
9974    validate_payload_codec(&task.payload_codec)?;
9975    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
9976    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
9977    for event in &task.history_events {
9978        validate_history_event_payloads(event, &task.payload_codec)?;
9979    }
9980
9981    let Some(export) = task.history_export.as_ref() else {
9982        return Ok(());
9983    };
9984    let export_codec = match export.get("payloads") {
9985        Some(payloads) => declared_payload_codec(payloads, "codec")?,
9986        None => None,
9987    }
9988    .unwrap_or(&task.payload_codec);
9989    validate_payload_codec(export_codec)?;
9990
9991    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
9992        for event in events {
9993            let event_type = event
9994                .get("event_type")
9995                .or_else(|| event.get("type"))
9996                .and_then(Value::as_str)
9997                .unwrap_or_default();
9998            if let Some(payload) = event.get("payload") {
9999                validate_history_payloads(event_type, payload, export_codec)?;
10000            }
10001        }
10002    }
10003    for signal in export
10004        .get("signals")
10005        .and_then(Value::as_array)
10006        .into_iter()
10007        .flatten()
10008    {
10009        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
10010        validate_payload_codec(codec)?;
10011        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
10012    }
10013    for activity in export
10014        .get("activities")
10015        .and_then(Value::as_array)
10016        .into_iter()
10017        .flatten()
10018    {
10019        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
10020        validate_payload_codec(codec)?;
10021        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
10022        validate_optional_inbound_payload(activity.get("result"), codec)?;
10023    }
10024    Ok(())
10025}
10026
10027fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
10028    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
10029}
10030
10031fn validate_history_payloads(
10032    event_type: &str,
10033    payload: &Value,
10034    fallback_codec: &str,
10035) -> Result<()> {
10036    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
10037    validate_payload_codec(codec)?;
10038    for field in history_payload_fields(event_type) {
10039        validate_optional_inbound_payload(payload.get(*field), codec)?;
10040    }
10041    Ok(())
10042}
10043
10044const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
10045
10046fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
10047    match event_type {
10048        "ActivityCompleted" => &["result"],
10049        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
10050        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
10051        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
10052        "ChildRunCompleted" => &["result", "output"],
10053        "WorkflowCompleted" => &["output"],
10054        "ServiceCallStarted"
10055        | "ServiceCallCompleted"
10056        | "ServiceCallFailed"
10057        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
10058        _ => &[],
10059    }
10060}
10061
10062fn signal_history_payload(payload: &Value) -> Option<&Value> {
10063    SIGNAL_HISTORY_PAYLOAD_FIELDS
10064        .iter()
10065        .find_map(|field| payload.get(*field))
10066}
10067
10068fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
10069    match value.get(field) {
10070        None => Ok(None),
10071        Some(Value::String(codec)) => Ok(Some(codec)),
10072        Some(_) => Err(invalid_payload_envelope()),
10073    }
10074}
10075
10076fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
10077    validate_payload_codec(codec)?;
10078    if let Some(value) = value.filter(|value| !value.is_null()) {
10079        decode_wire_avro_value(value, codec)?;
10080    }
10081    Ok(())
10082}
10083
10084fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
10085    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
10086        invalid_recorded_history(
10087            "parallel_group_metadata_invalid",
10088            sequence,
10089            "non-empty parallel_group_id",
10090            &payload.to_string(),
10091            "parallel-group history is missing its stable identity",
10092        )
10093    })?;
10094    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
10095        invalid_recorded_history(
10096            "parallel_group_metadata_invalid",
10097            sequence,
10098            "activity, child, timer, or mixed group kind",
10099            &payload.to_string(),
10100            "parallel-group history is missing its group kind",
10101        )
10102    })?;
10103    if !matches!(kind.as_str(), "activity" | "child" | "timer" | "mixed") {
10104        return Err(invalid_recorded_history(
10105            "parallel_group_metadata_invalid",
10106            sequence,
10107            "activity, child, timer, or mixed group kind",
10108            &kind,
10109            "parallel-group history contains an unsupported group kind",
10110        ));
10111    }
10112    let base_sequence = payload
10113        .get("parallel_group_base_sequence")
10114        .and_then(value_as_u64)
10115        .filter(|value| *value > 0)
10116        .ok_or_else(|| {
10117            invalid_recorded_history(
10118                "parallel_group_metadata_invalid",
10119                sequence,
10120                "positive parallel_group_base_sequence",
10121                &payload.to_string(),
10122                "parallel-group history contains an invalid base sequence",
10123            )
10124        })?;
10125    let size = payload
10126        .get("parallel_group_size")
10127        .and_then(value_as_u64)
10128        .and_then(|value| usize::try_from(value).ok())
10129        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
10130        .ok_or_else(|| {
10131            invalid_recorded_history(
10132                "parallel_group_metadata_invalid",
10133                sequence,
10134                "bounded positive parallel_group_size",
10135                &payload.to_string(),
10136                "parallel-group history contains an invalid group size",
10137            )
10138        })?;
10139    let index = payload
10140        .get("parallel_group_index")
10141        .and_then(value_as_u64)
10142        .and_then(|value| usize::try_from(value).ok())
10143        .filter(|value| *value < size)
10144        .ok_or_else(|| {
10145            invalid_recorded_history(
10146                "parallel_group_metadata_invalid",
10147                sequence,
10148                "parallel_group_index within group bounds",
10149                &payload.to_string(),
10150                "parallel-group history contains an invalid member index",
10151            )
10152        })?;
10153    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
10154        return Err(invalid_recorded_history(
10155            "parallel_group_metadata_invalid",
10156            sequence,
10157            "base sequence plus member index equals workflow sequence",
10158            &payload.to_string(),
10159            "parallel-group path does not preserve durable workflow position",
10160        ));
10161    }
10162    let expected_id = format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind));
10163    if group_id != expected_id {
10164        return Err(invalid_recorded_history(
10165            "parallel_group_metadata_invalid",
10166            sequence,
10167            &expected_id,
10168            &group_id,
10169            "parallel-group history contains an incompatible stable group ID",
10170        ));
10171    }
10172    Ok(ParallelGroupMetadata {
10173        parallel_group_id: group_id,
10174        parallel_group_kind: kind,
10175        parallel_group_base_sequence: base_sequence,
10176        parallel_group_size: size,
10177        parallel_group_index: index,
10178    })
10179}
10180
10181fn recorded_parallel_group_path(
10182    events: &[&HistoryEvent],
10183    sequence: u64,
10184) -> Result<Option<Vec<ParallelGroupMetadata>>> {
10185    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
10186    for event in events {
10187        let payload = &event.payload;
10188        let has_metadata = payload.get("parallel_group_path").is_some()
10189            || payload.get("parallel_group_id").is_some()
10190            || payload.get("parallel_group_kind").is_some()
10191            || payload.get("parallel_group_base_sequence").is_some()
10192            || payload.get("parallel_group_size").is_some()
10193            || payload.get("parallel_group_index").is_some();
10194        if !has_metadata {
10195            continue;
10196        }
10197
10198        let top_level = recorded_parallel_group_entry(payload, sequence)?;
10199        let path = match payload.get("parallel_group_path") {
10200            None => vec![top_level.clone()],
10201            Some(Value::Array(entries)) if !entries.is_empty() => entries
10202                .iter()
10203                .map(|entry| recorded_parallel_group_entry(entry, sequence))
10204                .collect::<Result<Vec<_>>>()?,
10205            Some(value) => {
10206                return Err(invalid_recorded_history(
10207                    "parallel_group_metadata_invalid",
10208                    sequence,
10209                    "non-empty parallel_group_path list",
10210                    &value.to_string(),
10211                    "parallel-group history contains an invalid group path",
10212                ));
10213            }
10214        };
10215        if path.last() != Some(&top_level) {
10216            return Err(invalid_recorded_history(
10217                "parallel_group_metadata_invalid",
10218                sequence,
10219                &serde_json::to_string(&path.last()).unwrap_or_default(),
10220                &serde_json::to_string(&top_level).unwrap_or_default(),
10221                "parallel-group top-level fields do not match the innermost path entry",
10222            ));
10223        }
10224        if recorded.as_ref().is_some_and(|existing| existing != &path) {
10225            return Err(invalid_recorded_history(
10226                "parallel_group_history_conflict",
10227                sequence,
10228                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
10229                &serde_json::to_string(&path).unwrap_or_default(),
10230                "parallel-group metadata changed between scheduling and resolution history",
10231            ));
10232        }
10233        recorded = Some(path);
10234    }
10235    Ok(recorded)
10236}
10237
10238fn recorded_commands(
10239    events: &[HistoryEvent],
10240    fallback_codec: &str,
10241    parent: WorkflowIdentity,
10242) -> Result<Vec<RecordedCommand>> {
10243    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
10244    let mut last_new_sequence = None;
10245
10246    for event in events {
10247        let is_activity = matches!(
10248            event.event_type.as_str(),
10249            "ActivityScheduled"
10250                | "ActivityStarted"
10251                | "ActivityHeartbeatRecorded"
10252                | "ActivityRetryScheduled"
10253                | "ActivityCompleted"
10254                | "ActivityFailed"
10255                | "ActivityCancelled"
10256                | "ActivityTimedOut"
10257        );
10258        let is_workflow_timer = matches!(
10259            event.event_type.as_str(),
10260            "TimerScheduled" | "TimerCancelled" | "TimerFired"
10261        ) && !is_internal_timer_event(event);
10262        let is_child_workflow = matches!(
10263            event.event_type.as_str(),
10264            "ChildWorkflowScheduled"
10265                | "ChildRunCompleted"
10266                | "ChildRunFailed"
10267                | "ChildRunCancelled"
10268                | "ChildRunTerminated"
10269        );
10270        let is_signal_wait = is_recorded_signal_wait_event(event);
10271        let is_condition_wait = is_recorded_condition_wait_event(event);
10272        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
10273        let is_side_effect = event.event_type == "SideEffectRecorded";
10274        let is_version_marker = event.event_type == "VersionMarkerRecorded";
10275        let is_memo = event.event_type == "MemoUpserted";
10276        if !is_activity
10277            && !is_workflow_timer
10278            && !is_child_workflow
10279            && !is_signal_wait
10280            && !is_condition_wait
10281            && !is_search_attributes
10282            && !is_side_effect
10283            && !is_version_marker
10284            && !is_memo
10285        {
10286            continue;
10287        }
10288
10289        let sequence = durable_event_sequence(event).ok_or_else(|| {
10290            Error::NonDeterministicReplay(ReplayFailure::new(
10291                "durable_command_sequence_missing",
10292                None,
10293                Some("positive workflow sequence".to_string()),
10294                Some(event.event_type.clone()),
10295                "durable command history event has no workflow sequence",
10296            ))
10297        })?;
10298        if sequence == 0 {
10299            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
10300                "durable_command_sequence_invalid",
10301                Some(sequence),
10302                Some("positive workflow sequence".to_string()),
10303                Some(sequence.to_string()),
10304                "durable command history uses an invalid workflow sequence",
10305            )));
10306        }
10307        if !events_by_sequence.contains_key(&sequence) {
10308            if let Some(previous) = last_new_sequence {
10309                if sequence < previous {
10310                    return Err(invalid_recorded_history(
10311                        "durable_command_sequence_mismatch",
10312                        sequence,
10313                        &format!("workflow sequence greater than {previous}"),
10314                        &sequence.to_string(),
10315                        "durable commands are not strictly ordered by their recorded workflow sequence",
10316                    ));
10317                }
10318            }
10319            last_new_sequence = Some(sequence);
10320        }
10321        events_by_sequence.entry(sequence).or_default().push(event);
10322    }
10323
10324    let commands: Vec<RecordedCommand> = events_by_sequence
10325        .into_iter()
10326        .map(|(sequence, sequence_events)| {
10327            let activity_events: Vec<_> = sequence_events
10328                .iter()
10329                .copied()
10330                .filter(|event| event.event_type.starts_with("Activity"))
10331                .collect();
10332            let timer_events: Vec<_> = sequence_events
10333                .iter()
10334                .copied()
10335                .filter(|event| event.event_type.starts_with("Timer"))
10336                .collect();
10337            let child_events: Vec<_> = sequence_events
10338                .iter()
10339                .copied()
10340                .filter(|event| {
10341                    event.event_type == "ChildWorkflowScheduled"
10342                        || event.event_type.starts_with("ChildRun")
10343                })
10344                .collect();
10345            let signal_wait_events: Vec<_> = sequence_events
10346                .iter()
10347                .copied()
10348                .filter(|event| is_recorded_signal_wait_event(event))
10349                .collect();
10350            let condition_wait_events: Vec<_> = sequence_events
10351                .iter()
10352                .copied()
10353                .filter(|event| is_recorded_condition_wait_event(event))
10354                .collect();
10355            let search_attribute_events: Vec<_> = sequence_events
10356                .iter()
10357                .copied()
10358                .filter(|event| event.event_type == "SearchAttributesUpserted")
10359                .collect();
10360            let side_effect_events: Vec<_> = sequence_events
10361                .iter()
10362                .copied()
10363                .filter(|event| event.event_type == "SideEffectRecorded")
10364                .collect();
10365            let version_marker_events: Vec<_> = sequence_events
10366                .iter()
10367                .copied()
10368                .filter(|event| event.event_type == "VersionMarkerRecorded")
10369                .collect();
10370            let memo_events: Vec<_> = sequence_events
10371                .iter()
10372                .copied()
10373                .filter(|event| event.event_type == "MemoUpserted")
10374                .collect();
10375
10376            let command_kind_count = usize::from(!activity_events.is_empty())
10377                + usize::from(!timer_events.is_empty())
10378                + usize::from(!child_events.is_empty())
10379                + usize::from(!signal_wait_events.is_empty())
10380                + usize::from(!condition_wait_events.is_empty())
10381                + usize::from(!search_attribute_events.is_empty())
10382                + usize::from(!side_effect_events.is_empty())
10383                + usize::from(!version_marker_events.is_empty())
10384                + usize::from(!memo_events.is_empty());
10385            if command_kind_count > 1 {
10386                let actual = [
10387                    (!activity_events.is_empty()).then_some("activity"),
10388                    (!timer_events.is_empty()).then_some("timer"),
10389                    (!child_events.is_empty()).then_some("child workflow"),
10390                    (!signal_wait_events.is_empty()).then_some("signal wait"),
10391                    (!condition_wait_events.is_empty()).then_some("condition wait"),
10392                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
10393                    (!side_effect_events.is_empty()).then_some("side effect"),
10394                    (!version_marker_events.is_empty()).then_some("version marker"),
10395                    (!memo_events.is_empty()).then_some("memo upsert"),
10396                ]
10397                .into_iter()
10398                .flatten()
10399                .collect::<Vec<_>>()
10400                .join(" and ");
10401                return Err(invalid_recorded_history(
10402                    "durable_command_sequence_collision",
10403                    sequence,
10404                    "one durable command kind",
10405                    &actual,
10406                    "one workflow sequence records more than one durable command kind",
10407                ));
10408            }
10409
10410            if !activity_events.is_empty() {
10411                let parallel_group_path =
10412                    recorded_parallel_group_path(&activity_events, sequence)?;
10413                let scheduled_count = activity_events
10414                    .iter()
10415                    .filter(|event| event.event_type == "ActivityScheduled")
10416                    .count();
10417                if scheduled_count > 1 {
10418                    return Err(invalid_recorded_history(
10419                        "duplicate_activity_schedule",
10420                        sequence,
10421                        "at most one ActivityScheduled event",
10422                        "multiple ActivityScheduled events",
10423                        "activity history schedules more than one command at one workflow sequence",
10424                    ));
10425                }
10426                let activity_type = activity_events.iter().find_map(|event| {
10427                    event
10428                        .payload
10429                        .get("activity_type")
10430                        .or_else(|| event.payload.get("activity_name"))
10431                        .and_then(Value::as_str)
10432                        .map(str::to_string)
10433                });
10434                if activity_events.iter().filter_map(|event| {
10435                    event
10436                        .payload
10437                        .get("activity_type")
10438                        .or_else(|| event.payload.get("activity_name"))
10439                        .and_then(Value::as_str)
10440                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
10441                    return Err(invalid_recorded_history(
10442                        "activity_identity_mismatch",
10443                        sequence,
10444                        activity_type.as_deref().unwrap_or("one activity identity"),
10445                        "conflicting activity identities",
10446                        "activity lifecycle events at one workflow sequence disagree on identity",
10447                    ));
10448                }
10449                let terminal: Vec<_> = activity_events
10450                    .iter()
10451                    .copied()
10452                    .filter(|event| {
10453                        matches!(
10454                            event.event_type.as_str(),
10455                            "ActivityCompleted"
10456                                | "ActivityFailed"
10457                                | "ActivityCancelled"
10458                                | "ActivityTimedOut"
10459                        )
10460                    })
10461                    .collect();
10462                let duplicate_delivery = terminal.first().is_some_and(|first| {
10463                    terminal.iter().all(|event| {
10464                        event.event_type == first.event_type && event.payload == first.payload
10465                    })
10466                });
10467                if terminal.len() > 1 && !duplicate_delivery {
10468                    return Err(invalid_recorded_history(
10469                        "duplicate_activity_terminal_event",
10470                        sequence,
10471                        "at most one terminal activity event",
10472                        "multiple terminal activity events",
10473                        "activity history settles one command more than once",
10474                    ));
10475                }
10476                let outcome = terminal
10477                    .first()
10478                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
10479                    .transpose()?;
10480                let options = activity_events
10481                    .iter()
10482                    .find(|event| event.event_type == "ActivityScheduled")
10483                    .and_then(|event| event.payload.get("activity"))
10484                    .and_then(Value::as_object)
10485                    .map(|activity| RecordedActivityOptions {
10486                        task_queue: recorded_optional_string(activity, "queue"),
10487                        execution_mode: recorded_optional_string(activity, "execution_mode"),
10488                        retry_policy: recorded_activity_retry_snapshot(
10489                            activity.get("retry_policy"),
10490                        ),
10491                    });
10492                return Ok(RecordedCommand::Activity {
10493                    sequence,
10494                    activity_type,
10495                    options,
10496                    outcome,
10497                    parallel_group_path,
10498                });
10499            }
10500
10501            if !child_events.is_empty() {
10502                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
10503                let scheduled: Vec<_> = child_events
10504                    .iter()
10505                    .copied()
10506                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
10507                    .collect();
10508                if scheduled.len() != 1 {
10509                    return Err(invalid_recorded_history(
10510                        "child_workflow_schedule_missing_or_duplicate",
10511                        sequence,
10512                        "one ChildWorkflowScheduled event",
10513                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
10514                        "child workflow replay requires exactly one recorded schedule event",
10515                    ));
10516                }
10517                let workflow_type = child_events.iter().find_map(|event| {
10518                    event
10519                        .payload
10520                        .get("child_workflow_type")
10521                        .or_else(|| event.payload.get("workflow_type"))
10522                        .and_then(Value::as_str)
10523                        .filter(|value| !value.is_empty())
10524                        .map(str::to_string)
10525                });
10526                if child_events
10527                    .iter()
10528                    .filter_map(|event| {
10529                        event
10530                            .payload
10531                            .get("child_workflow_type")
10532                            .or_else(|| event.payload.get("workflow_type"))
10533                            .and_then(Value::as_str)
10534                    })
10535                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
10536                {
10537                    return Err(invalid_recorded_history(
10538                        "child_workflow_identity_mismatch",
10539                        sequence,
10540                        workflow_type
10541                            .as_deref()
10542                            .unwrap_or("one child workflow type"),
10543                        "conflicting child workflow types",
10544                        "child workflow lifecycle events at one sequence disagree on type",
10545                    ));
10546                }
10547                let mut outcomes = child_workflow_outcomes(
10548                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
10549                    fallback_codec,
10550                    parent.clone(),
10551                )?;
10552                let terminal_events = child_events
10553                    .iter()
10554                    .copied()
10555                    .filter(|event| event.event_type.starts_with("ChildRun"))
10556                    .collect::<Vec<_>>();
10557                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
10558                    terminal_events.iter().all(|event| {
10559                        event.event_type == first.event_type && event.payload == first.payload
10560                    })
10561                });
10562                if outcomes.len() > 1 && !duplicate_delivery {
10563                    return Err(invalid_recorded_history(
10564                        "duplicate_child_workflow_terminal_event",
10565                        sequence,
10566                        "at most one terminal child event",
10567                        "multiple terminal child events",
10568                        "child workflow history settles one command more than once",
10569                    ));
10570                }
10571                return Ok(RecordedCommand::ChildWorkflow {
10572                    sequence,
10573                    workflow_type,
10574                    outcome: outcomes.pop(),
10575                    parallel_group_path,
10576                });
10577            }
10578
10579            if !signal_wait_events.is_empty() {
10580                let opened: Vec<_> = signal_wait_events
10581                    .iter()
10582                    .copied()
10583                    .filter(|event| event.event_type == "SignalWaitOpened")
10584                    .collect();
10585                if opened.len() != 1 {
10586                    return Err(invalid_recorded_history(
10587                        "signal_wait_open_missing_or_duplicate",
10588                        sequence,
10589                        "one SignalWaitOpened event",
10590                        &format!("{} SignalWaitOpened events", opened.len()),
10591                        "signal replay requires exactly one canonical wait-open event",
10592                    ));
10593                }
10594
10595                let applied: Vec<_> = signal_wait_events
10596                    .iter()
10597                    .copied()
10598                    .filter(|event| event.event_type == "SignalApplied")
10599                    .collect();
10600                if applied.len() > 1 {
10601                    return Err(invalid_recorded_history(
10602                        "duplicate_signal_wait_apply",
10603                        sequence,
10604                        "at most one SignalApplied event",
10605                        "multiple SignalApplied events",
10606                        "signal history applies one durable wait more than once",
10607                    ));
10608                }
10609
10610                let signal_names = signal_wait_events
10611                    .iter()
10612                    .map(|event| required_signal_wait_name(event, sequence))
10613                    .collect::<Result<Vec<_>>>()?;
10614                let signal_name = signal_names
10615                    .first()
10616                    .expect("signal wait events are not empty")
10617                    .clone();
10618                if signal_names.iter().any(|candidate| candidate != &signal_name) {
10619                    return Err(invalid_recorded_history(
10620                        "signal_wait_identity_mismatch",
10621                        sequence,
10622                        &signal_name,
10623                        "conflicting signal names",
10624                        "signal wait lifecycle events at one workflow sequence disagree on identity",
10625                    ));
10626                }
10627                let value = applied
10628                    .first()
10629                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
10630                    .transpose()?;
10631                return Ok(RecordedCommand::SignalWait {
10632                    sequence,
10633                    signal_name,
10634                    value,
10635                });
10636            }
10637
10638            if !condition_wait_events.is_empty() {
10639                return recorded_condition_wait(
10640                    sequence,
10641                    &condition_wait_events,
10642                    events,
10643                );
10644            }
10645
10646            if !search_attribute_events.is_empty() {
10647                if search_attribute_events.len() != 1 {
10648                    return Err(invalid_recorded_history(
10649                        "duplicate_search_attribute_update",
10650                        sequence,
10651                        "one SearchAttributesUpserted event",
10652                        &format!(
10653                            "{} SearchAttributesUpserted events",
10654                            search_attribute_events.len()
10655                        ),
10656                        "search-attribute history records one workflow command more than once",
10657                    ));
10658                }
10659                let payload = &search_attribute_events[0].payload;
10660                let attributes = payload
10661                    .get("attributes")
10662                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
10663                    .cloned()
10664                    .ok_or_else(|| {
10665                        invalid_recorded_history(
10666                            "search_attribute_update_missing",
10667                            sequence,
10668                            "non-empty attributes object",
10669                            "missing or invalid attributes",
10670                            "search-attribute history is missing its recorded mutation",
10671                        )
10672                    })?;
10673                let attribute_types =
10674                    recorded_search_attribute_types(payload, &attributes, sequence)?;
10675                return Ok(RecordedCommand::SearchAttributes {
10676                    sequence,
10677                    attributes,
10678                    attribute_types,
10679                });
10680            }
10681
10682            if !side_effect_events.is_empty() {
10683                if side_effect_events.len() != 1 {
10684                    return Err(invalid_recorded_history(
10685                        "duplicate_side_effect_record",
10686                        sequence,
10687                        "one SideEffectRecorded event",
10688                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
10689                        "side-effect history records one workflow command more than once",
10690                    ));
10691                }
10692                let event = side_effect_events[0];
10693                let result = event.payload.get("result").ok_or_else(|| {
10694                    invalid_recorded_history(
10695                        "side_effect_result_missing",
10696                        sequence,
10697                        "recorded result payload",
10698                        "missing result",
10699                        "side-effect history is missing its recorded value",
10700                    )
10701                })?;
10702                let has_published_envelope = result.as_str().is_some()
10703                    || result.as_object().is_some_and(|envelope| {
10704                        envelope.get("codec").and_then(Value::as_str).is_some()
10705                            && envelope.get("blob").and_then(Value::as_str).is_some()
10706                    });
10707                if !has_published_envelope {
10708                    return Err(invalid_recorded_history(
10709                        "side_effect_payload_malformed",
10710                        sequence,
10711                        "payload blob or {codec, blob} envelope",
10712                        &result.to_string(),
10713                        "side-effect history result does not use a published payload envelope",
10714                    ));
10715                }
10716                let codec = event
10717                    .payload
10718                    .get("payload_codec")
10719                    .and_then(Value::as_str)
10720                    .unwrap_or(fallback_codec);
10721                let value = decode_wire_avro_value(result, codec).map_err(|error| {
10722                    if error.to_string().contains("unsupported_payload_codec") {
10723                        return error;
10724                    }
10725
10726                    invalid_recorded_history(
10727                        "side_effect_payload_incompatible",
10728                        sequence,
10729                        &format!("valid {codec} payload envelope"),
10730                        &error.to_string(),
10731                        "side-effect history payload cannot be decoded with its recorded codec",
10732                    )
10733                })?;
10734                return Ok(RecordedCommand::SideEffect { sequence, value });
10735            }
10736
10737            if !version_marker_events.is_empty() {
10738                if version_marker_events.len() != 1 {
10739                    return Err(invalid_recorded_history(
10740                        "duplicate_version_marker_record",
10741                        sequence,
10742                        "one VersionMarkerRecorded event",
10743                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
10744                        "version-marker history records one workflow command more than once",
10745                    ));
10746                }
10747                let payload = &version_marker_events[0].payload;
10748                let change_id = payload
10749                    .get("change_id")
10750                    .and_then(Value::as_str)
10751                    .filter(|value| !value.is_empty())
10752                    .map(str::to_string)
10753                    .ok_or_else(|| {
10754                        invalid_recorded_history(
10755                            "version_marker_field_missing",
10756                            sequence,
10757                            "non-empty change_id",
10758                            "missing or invalid change_id",
10759                            "version-marker history is missing its stable change ID",
10760                        )
10761                    })?;
10762                let version = required_version_i32(payload, "version", sequence)?;
10763                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
10764                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
10765                if min_supported > max_supported || version < min_supported || version > max_supported {
10766                    return Err(invalid_recorded_history(
10767                        "version_marker_history_range_invalid",
10768                        sequence,
10769                        "min_supported <= version <= max_supported",
10770                        &format!("{min_supported} <= {version} <= {max_supported}"),
10771                        "recorded version marker contains an internally incompatible range",
10772                    ));
10773                }
10774                return Ok(RecordedCommand::VersionMarker {
10775                    sequence,
10776                    change_id,
10777                    version,
10778                });
10779            }
10780
10781            if !memo_events.is_empty() {
10782                if memo_events.len() != 1 {
10783                    return Err(invalid_recorded_history(
10784                        "duplicate_memo_upsert_record",
10785                        sequence,
10786                        "one MemoUpserted event",
10787                        &format!("{} MemoUpserted events", memo_events.len()),
10788                        "memo history records one workflow update more than once",
10789                    ));
10790                }
10791                let payload = &memo_events[0].payload;
10792                let entries = payload.get("entries").cloned().ok_or_else(|| {
10793                    invalid_recorded_history(
10794                        "memo_entries_missing",
10795                        sequence,
10796                        "memo entries object",
10797                        "missing entries",
10798                        "MemoUpserted history is missing replay identity entries",
10799                    )
10800                })?;
10801                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
10802                    invalid_recorded_history(
10803                        "memo_entries_invalid",
10804                        sequence,
10805                        "valid canonical memo entries",
10806                        &error.to_string(),
10807                        "MemoUpserted history contains invalid replay identity entries",
10808                    )
10809                })?;
10810                let merged = payload.get("merged").cloned().ok_or_else(|| {
10811                    invalid_recorded_history(
10812                        "memo_merged_projection_missing",
10813                        sequence,
10814                        "merged memo projection",
10815                        "missing merged",
10816                        "MemoUpserted history is missing its merged projection",
10817                    )
10818                })?;
10819                decode_memo_history_map(&merged, false).map_err(|error| {
10820                    invalid_recorded_history(
10821                        "memo_merged_projection_invalid",
10822                        sequence,
10823                        "valid merged memo projection",
10824                        &error.to_string(),
10825                        "MemoUpserted history contains an invalid merged projection",
10826                    )
10827                })?;
10828
10829                return Ok(RecordedCommand::Memo { sequence, entries });
10830            }
10831            let scheduled: Vec<_> = timer_events
10832                .iter()
10833                .copied()
10834                .filter(|event| event.event_type == "TimerScheduled")
10835                .collect();
10836            let fired: Vec<_> = timer_events
10837                .iter()
10838                .copied()
10839                .filter(|event| event.event_type == "TimerFired")
10840                .collect();
10841            if scheduled.len() != 1 {
10842                return Err(invalid_recorded_history(
10843                    "timer_schedule_missing_or_duplicate",
10844                    sequence,
10845                    "one TimerScheduled event",
10846                    &format!("{} TimerScheduled events", scheduled.len()),
10847                    "timer replay requires exactly one recorded schedule event",
10848                ));
10849            }
10850            if fired.len() > 1 {
10851                return Err(invalid_recorded_history(
10852                    "duplicate_timer_fire",
10853                    sequence,
10854                    "at most one TimerFired event",
10855                    "multiple TimerFired events",
10856                    "timer history contains more than one fire event for a workflow sequence",
10857                ));
10858            }
10859
10860            let scheduled = scheduled[0];
10861            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
10862            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
10863            if let Some(fired) = fired.first() {
10864                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
10865                if fired_timer_id != timer_id {
10866                    return Err(invalid_recorded_history(
10867                        "timer_identity_mismatch",
10868                        sequence,
10869                        &timer_id,
10870                        &fired_timer_id,
10871                        "TimerFired does not correspond to the recorded TimerScheduled event",
10872                    ));
10873                }
10874                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
10875                if fired_delay != delay_seconds {
10876                    return Err(invalid_recorded_history(
10877                        "timer_history_delay_mismatch",
10878                        sequence,
10879                        &delay_seconds.to_string(),
10880                        &fired_delay.to_string(),
10881                        "TimerScheduled and TimerFired record different delays",
10882                    ));
10883                }
10884            }
10885
10886            Ok(RecordedCommand::Timer {
10887                sequence,
10888                delay_seconds,
10889                fired: !fired.is_empty(),
10890                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
10891            })
10892        })
10893        .collect::<Result<_>>()?;
10894
10895    let mut marker_sequences = HashMap::new();
10896    for command in &commands {
10897        if let RecordedCommand::VersionMarker {
10898            sequence,
10899            change_id,
10900            ..
10901        } = command
10902        {
10903            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
10904                return Err(invalid_recorded_history(
10905                    "duplicate_version_marker",
10906                    *sequence,
10907                    &format!("one marker for change ID {change_id:?}"),
10908                    &format!("markers at sequences {first_sequence} and {sequence}"),
10909                    "workflow history contains duplicate markers for one stable change ID",
10910                ));
10911            }
10912        }
10913    }
10914
10915    Ok(commands)
10916}
10917
10918fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
10919    payload
10920        .get(field)
10921        .and_then(Value::as_i64)
10922        .and_then(|value| i32::try_from(value).ok())
10923        .ok_or_else(|| {
10924            invalid_recorded_history(
10925                "version_marker_field_missing",
10926                sequence,
10927                &format!("integer {field}"),
10928                "missing or out-of-range integer",
10929                "version-marker history is missing a required integer field",
10930            )
10931        })
10932}
10933
10934fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
10935    event
10936        .payload
10937        .get("sequence")
10938        .or_else(|| event.payload.get("workflow_sequence"))
10939        .or_else(|| event.raw.get("sequence"))
10940        .or_else(|| event.raw.get("workflow_sequence"))
10941        .and_then(value_as_u64)
10942}
10943
10944fn is_internal_timer_event(event: &HistoryEvent) -> bool {
10945    matches!(
10946        event
10947            .payload
10948            .get("timer_kind")
10949            .or_else(|| event.raw.get("timer_kind"))
10950            .and_then(Value::as_str),
10951        Some("condition_timeout" | "signal_timeout")
10952    )
10953}
10954
10955fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
10956    matches!(
10957        event.event_type.as_str(),
10958        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
10959    )
10960}
10961
10962fn recorded_condition_wait(
10963    sequence: u64,
10964    condition_events: &[&HistoryEvent],
10965    all_events: &[HistoryEvent],
10966) -> Result<RecordedCommand> {
10967    let opened = condition_events
10968        .iter()
10969        .copied()
10970        .filter(|event| event.event_type == "ConditionWaitOpened")
10971        .collect::<Vec<_>>();
10972    if opened.len() != 1 {
10973        return Err(invalid_recorded_history(
10974            "condition_wait_open_missing_or_duplicate",
10975            sequence,
10976            "one ConditionWaitOpened event",
10977            &format!("{} ConditionWaitOpened events", opened.len()),
10978            "condition replay requires exactly one canonical wait-open event",
10979        ));
10980    }
10981    let terminal = condition_events
10982        .iter()
10983        .copied()
10984        .filter(|event| {
10985            matches!(
10986                event.event_type.as_str(),
10987                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
10988            )
10989        })
10990        .collect::<Vec<_>>();
10991    if terminal.len() > 1 {
10992        return Err(invalid_recorded_history(
10993            "duplicate_condition_wait_terminal_event",
10994            sequence,
10995            "at most one condition terminal event",
10996            "multiple condition terminal events",
10997            "condition history settles one durable wait more than once",
10998        ));
10999    }
11000
11001    let opened = opened[0];
11002    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
11003    for event in condition_events
11004        .iter()
11005        .copied()
11006        .filter(|event| !std::ptr::eq(*event, opened))
11007    {
11008        let event_wait_id = required_condition_wait_id(event, sequence)?;
11009        if event_wait_id != condition_wait_id {
11010            return Err(invalid_recorded_history(
11011                "condition_wait_id_mismatch",
11012                sequence,
11013                &condition_wait_id,
11014                &event_wait_id,
11015                "condition lifecycle events at one sequence disagree on wait identity",
11016            ));
11017        }
11018    }
11019
11020    let condition_key = optional_non_empty_history_string(opened, "condition_key");
11021    let predicate_identity = opened
11022        .payload
11023        .get("condition_definition_fingerprint")
11024        .and_then(Value::as_str)
11025        .filter(|value| !value.is_empty())
11026        .map(str::to_string)
11027        .ok_or_else(|| {
11028            invalid_recorded_history(
11029                "condition_wait_predicate_fingerprint_missing",
11030                sequence,
11031                "non-empty condition_definition_fingerprint",
11032                &opened.event_type,
11033                "canonical condition history is missing its predicate identity",
11034            )
11035        })?;
11036    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
11037    for event in condition_events
11038        .iter()
11039        .copied()
11040        .filter(|event| !std::ptr::eq(*event, opened))
11041    {
11042        for (field, opened_value) in [
11043            ("condition_key", condition_key.as_deref()),
11044            (
11045                "condition_definition_fingerprint",
11046                Some(predicate_identity.as_str()),
11047            ),
11048        ] {
11049            if let Some(value) = optional_non_empty_history_string(event, field) {
11050                if opened_value.is_some_and(|opened_value| opened_value != value) {
11051                    return Err(invalid_recorded_history(
11052                        "condition_wait_definition_history_mismatch",
11053                        sequence,
11054                        opened_value.unwrap_or_default(),
11055                        &value,
11056                        "condition lifecycle events disagree on the recorded definition",
11057                    ));
11058                }
11059            }
11060        }
11061        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
11062            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
11063                return Err(invalid_recorded_history(
11064                    "condition_wait_definition_history_mismatch",
11065                    sequence,
11066                    &format!("{}s", timeout_seconds.unwrap_or_default()),
11067                    &format!("{event_timeout}s"),
11068                    "condition lifecycle events disagree on the recorded timeout",
11069                ));
11070            }
11071        }
11072    }
11073
11074    let timeout_timer_events = all_events
11075        .iter()
11076        .filter(|event| {
11077            matches!(
11078                event.event_type.as_str(),
11079                "TimerScheduled" | "TimerCancelled" | "TimerFired"
11080            ) && event.payload.get("timer_kind").and_then(Value::as_str)
11081                == Some("condition_timeout")
11082                && event
11083                    .payload
11084                    .get("condition_wait_id")
11085                    .and_then(Value::as_str)
11086                    == Some(condition_wait_id.as_str())
11087        })
11088        .collect::<Vec<_>>();
11089    let scheduled = timeout_timer_events
11090        .iter()
11091        .copied()
11092        .filter(|event| event.event_type == "TimerScheduled")
11093        .collect::<Vec<_>>();
11094    let fired = timeout_timer_events
11095        .iter()
11096        .copied()
11097        .filter(|event| event.event_type == "TimerFired")
11098        .collect::<Vec<_>>();
11099    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
11100        return Err(invalid_recorded_history(
11101            "condition_wait_timeout_history_invalid",
11102            sequence,
11103            "one timeout schedule and at most one fire",
11104            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
11105            "condition timeout history has a missing or duplicate lifecycle event",
11106        ));
11107    }
11108    if let Some(scheduled) = scheduled.first() {
11109        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
11110        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
11111        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
11112            return Err(invalid_recorded_history(
11113                "condition_wait_timeout_delay_mismatch",
11114                sequence,
11115                &format!("{}s", timeout_seconds.unwrap_or_default()),
11116                &format!("{delay_seconds}s"),
11117                "condition timeout timer differs from the wait definition",
11118            ));
11119        }
11120        if let Some(fired) = fired.first() {
11121            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
11122            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
11123            if fired_timer_id != timer_id || fired_delay != delay_seconds {
11124                return Err(invalid_recorded_history(
11125                    "condition_wait_timeout_identity_mismatch",
11126                    sequence,
11127                    &format!("{timer_id}:{delay_seconds}s"),
11128                    &format!("{fired_timer_id}:{fired_delay}s"),
11129                    "condition timeout fire does not match its durable schedule",
11130                ));
11131            }
11132        }
11133    }
11134
11135    let result = terminal.first().map(|event| {
11136        if event.event_type == "ConditionWaitTimedOut" {
11137            ConditionWaitResult::TimedOut
11138        } else {
11139            ConditionWaitResult::Satisfied
11140        }
11141    });
11142    let result = if !fired.is_empty() {
11143        if result == Some(ConditionWaitResult::Satisfied) {
11144            return Err(invalid_recorded_history(
11145                "condition_wait_terminal_conflict",
11146                sequence,
11147                "one satisfied or timed-out outcome",
11148                "satisfied event and fired timeout",
11149                "condition history records conflicting terminal outcomes",
11150            ));
11151        }
11152        Some(ConditionWaitResult::TimedOut)
11153    } else {
11154        result
11155    };
11156
11157    Ok(RecordedCommand::ConditionWait {
11158        sequence,
11159        condition_key,
11160        predicate_identity,
11161        timeout_seconds,
11162        result,
11163    })
11164}
11165
11166fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
11167    event
11168        .payload
11169        .get("condition_wait_id")
11170        .and_then(Value::as_str)
11171        .filter(|value| !value.is_empty())
11172        .map(str::to_string)
11173        .ok_or_else(|| {
11174            invalid_recorded_history(
11175                "condition_wait_id_missing",
11176                sequence,
11177                "non-empty condition_wait_id",
11178                &event.event_type,
11179                "canonical condition history is missing its durable wait identity",
11180            )
11181        })
11182}
11183
11184fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
11185    event
11186        .payload
11187        .get(field)
11188        .and_then(Value::as_str)
11189        .filter(|value| !value.is_empty())
11190        .map(str::to_string)
11191}
11192
11193fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
11194    match event.payload.get(field) {
11195        None | Some(Value::Null) => Ok(None),
11196        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
11197            invalid_recorded_history(
11198                "condition_wait_definition_invalid",
11199                sequence,
11200                &format!("non-negative integer {field}"),
11201                &value.to_string(),
11202                "condition history contains an invalid numeric definition field",
11203            )
11204        }),
11205    }
11206}
11207
11208fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
11209    event
11210        .payload
11211        .get("signal_name")
11212        .or_else(|| event.raw.get("signal_name"))
11213        .and_then(Value::as_str)
11214        .filter(|value| !value.is_empty())
11215        .map(str::to_string)
11216        .ok_or_else(|| {
11217            invalid_recorded_history(
11218                "signal_wait_name_missing",
11219                sequence,
11220                "non-empty signal_name",
11221                &event.event_type,
11222                "canonical signal-wait history is missing its signal identity",
11223            )
11224        })
11225}
11226
11227fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
11228    matches!(
11229        event.event_type.as_str(),
11230        "SignalWaitOpened" | "SignalApplied"
11231    )
11232}
11233
11234fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
11235    event
11236        .payload
11237        .get(field)
11238        .and_then(Value::as_str)
11239        .filter(|value| !value.is_empty())
11240        .map(str::to_string)
11241        .ok_or_else(|| {
11242            invalid_recorded_history(
11243                "timer_history_field_missing",
11244                sequence,
11245                field,
11246                &event.event_type,
11247                "timer history is missing a required identity field",
11248            )
11249        })
11250}
11251
11252fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
11253    event
11254        .payload
11255        .get(field)
11256        .and_then(value_as_u64)
11257        .ok_or_else(|| {
11258            invalid_recorded_history(
11259                "timer_history_field_missing",
11260                sequence,
11261                field,
11262                &event.event_type,
11263                "timer history is missing a required numeric field",
11264            )
11265        })
11266}
11267
11268fn recorded_search_attribute_types(
11269    payload: &Value,
11270    attributes: &Value,
11271    sequence: u64,
11272) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
11273    let Some(raw_types) = payload.get("attribute_types") else {
11274        // This is the explicit compatibility rule for histories recorded
11275        // before typed identity was persisted. Values still constrain replay;
11276        // the unknown type snapshot does not assert a typed match.
11277        return Ok(RecordedSnapshotValue::Unknown);
11278    };
11279    let Some(raw_types) = raw_types.as_object() else {
11280        return Err(invalid_recorded_history(
11281            "search_attribute_types_malformed",
11282            sequence,
11283            "canonical attribute type map",
11284            &raw_types.to_string(),
11285            "search-attribute history contains malformed type identity",
11286        ));
11287    };
11288    let attribute_keys = attributes
11289        .as_object()
11290        .expect("recorded search attributes were validated as an object");
11291    let mut types = BTreeMap::new();
11292    for (key, value) in raw_types {
11293        let Some(attribute_type) = value.as_str() else {
11294            return Err(invalid_recorded_history(
11295                "search_attribute_types_malformed",
11296                sequence,
11297                "canonical string type name",
11298                &value.to_string(),
11299                "search-attribute history contains a non-string type identity",
11300            ));
11301        };
11302        if !attribute_keys.contains_key(key)
11303            || !matches!(
11304                attribute_type,
11305                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
11306            )
11307        {
11308            return Err(invalid_recorded_history(
11309                "search_attribute_types_malformed",
11310                sequence,
11311                "canonical types for keys present in attributes",
11312                &format!("{key}:{attribute_type}"),
11313                "search-attribute history contains unsupported or orphaned type identity",
11314            ));
11315        }
11316        types.insert(key.clone(), attribute_type.to_string());
11317    }
11318    Ok(RecordedSnapshotValue::Known(types))
11319}
11320
11321fn invalid_recorded_history(
11322    reason: &str,
11323    sequence: u64,
11324    expected: &str,
11325    actual: &str,
11326    message: &str,
11327) -> Error {
11328    Error::NonDeterministicReplay(ReplayFailure::new(
11329        reason,
11330        Some(sequence),
11331        Some(expected.to_string()),
11332        Some(actual.to_string()),
11333        message,
11334    ))
11335}
11336
11337type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
11338
11339fn activity_outcome(
11340    event: &HistoryEvent,
11341    fallback_codec: &str,
11342    recorded_activity_type: Option<String>,
11343) -> Result<ActivityOutcome> {
11344    if event.event_type == "ActivityCompleted" {
11345        let codec = event
11346            .payload
11347            .get("payload_codec")
11348            .and_then(Value::as_str)
11349            .unwrap_or(fallback_codec);
11350        return Ok(Ok(decode_wire_avro_value(
11351            event.payload.get("result").unwrap_or(&Value::Null),
11352            codec,
11353        )?));
11354    }
11355
11356    let payload = &event.payload;
11357    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
11358        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
11359        "ActivityCancelled" => (
11360            ActivityFailureKind::Cancelled,
11361            "cancelled",
11362            "activity was cancelled",
11363        ),
11364        "ActivityTimedOut" => (
11365            ActivityFailureKind::TimedOut,
11366            "timeout",
11367            "activity timed out",
11368        ),
11369        _ => unreachable!("activity_outcome is called only for terminal activity events"),
11370    };
11371    let exception = payload
11372        .get("exception")
11373        .filter(|value| !value.is_null())
11374        .cloned();
11375    let failure_category = payload_string(payload, "failure_category");
11376    let timeout_kind = payload_string(payload, "timeout_kind");
11377    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
11378        ActivityFailureKind::Failed => failure_category
11379            .clone()
11380            .unwrap_or_else(|| fallback_reason.to_string()),
11381        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
11382        ActivityFailureKind::TimedOut => timeout_kind
11383            .clone()
11384            .unwrap_or_else(|| fallback_reason.to_string()),
11385    });
11386    let message = payload_string(payload, "message")
11387        .or_else(|| {
11388            exception
11389                .as_ref()
11390                .and_then(|value| payload_string(value, "message"))
11391        })
11392        .unwrap_or_else(|| fallback_message.to_string());
11393
11394    Ok(Err(ActivityFailure {
11395        kind,
11396        reason,
11397        message,
11398        activity_execution_id: payload_string(payload, "activity_execution_id"),
11399        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
11400        activity_type: payload_string(payload, "activity_type")
11401            .or_else(|| payload_string(payload, "activity_name"))
11402            .or(recorded_activity_type),
11403        activity_class: payload_string(payload, "activity_class"),
11404        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
11405        failure_id: payload_string(payload, "failure_id"),
11406        failure_category,
11407        timeout_kind,
11408        non_retryable: payload
11409            .get("non_retryable")
11410            .and_then(Value::as_bool)
11411            .unwrap_or(false),
11412        exception_type: payload_string(payload, "exception_type").or_else(|| {
11413            exception
11414                .as_ref()
11415                .and_then(|value| payload_string(value, "type"))
11416        }),
11417        exception_class: payload_string(payload, "exception_class").or_else(|| {
11418            exception
11419                .as_ref()
11420                .and_then(|value| payload_string(value, "class"))
11421        }),
11422        code: payload
11423            .get("code")
11424            .filter(|value| !value.is_null())
11425            .cloned(),
11426        exception,
11427    }))
11428}
11429
11430type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
11431
11432fn child_workflow_outcomes(
11433    events: &[HistoryEvent],
11434    fallback_codec: &str,
11435    parent: WorkflowIdentity,
11436) -> Result<Vec<ChildWorkflowOutcome>> {
11437    let mut outcomes = Vec::new();
11438
11439    for event in events {
11440        let kind = match event.event_type.as_str() {
11441            "ChildRunCompleted" => None,
11442            "ChildRunFailed" => Some((
11443                ChildWorkflowFailureKind::Failed,
11444                "child_workflow",
11445                "child workflow failed",
11446            )),
11447            "ChildRunCancelled" => Some((
11448                ChildWorkflowFailureKind::Cancelled,
11449                "cancelled",
11450                "child workflow was cancelled",
11451            )),
11452            "ChildRunTerminated" => Some((
11453                ChildWorkflowFailureKind::Terminated,
11454                "terminated",
11455                "child workflow was terminated",
11456            )),
11457            _ => continue,
11458        };
11459        let payload = &event.payload;
11460        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
11461        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
11462        let child_workflow_type = payload_string(payload, "child_workflow_type");
11463
11464        if let Some((kind, reason, fallback_message)) = kind {
11465            let exception = payload
11466                .get("exception")
11467                .filter(|value| !value.is_null())
11468                .cloned();
11469            let message = payload_string(payload, "message")
11470                .or_else(|| {
11471                    exception
11472                        .as_ref()
11473                        .and_then(|value| payload_string(value, "message"))
11474                })
11475                .unwrap_or_else(|| fallback_message.to_string());
11476            let exception_type = payload_string(payload, "exception_type").or_else(|| {
11477                exception
11478                    .as_ref()
11479                    .and_then(|value| payload_string(value, "type"))
11480            });
11481            let exception_class = payload_string(payload, "exception_class").or_else(|| {
11482                exception
11483                    .as_ref()
11484                    .and_then(|value| payload_string(value, "class"))
11485            });
11486            outcomes.push(Err(ChildWorkflowFailure {
11487                kind,
11488                reason: reason.to_string(),
11489                message,
11490                parent_workflow_id: parent.workflow_id.clone(),
11491                parent_workflow_run_id: parent.run_id.clone(),
11492                child_workflow_id,
11493                child_workflow_run_id,
11494                child_workflow_type,
11495                failure_id: payload_string(payload, "failure_id"),
11496                failure_category: payload_string(payload, "failure_category"),
11497                exception_type,
11498                exception_class,
11499                non_retryable: payload
11500                    .get("non_retryable")
11501                    .and_then(Value::as_bool)
11502                    .unwrap_or(false),
11503                code: payload
11504                    .get("code")
11505                    .filter(|value| !value.is_null())
11506                    .cloned(),
11507                exception,
11508            }));
11509            continue;
11510        }
11511
11512        let codec = payload
11513            .get("payload_codec")
11514            .and_then(Value::as_str)
11515            .unwrap_or(fallback_codec);
11516        let result = payload
11517            .get("result")
11518            .or_else(|| payload.get("output"))
11519            .unwrap_or(&Value::Null);
11520        outcomes.push(Ok(ChildWorkflowAvroResult {
11521            parent: parent.clone(),
11522            child: WorkflowIdentity {
11523                workflow_id: child_workflow_id,
11524                run_id: child_workflow_run_id,
11525            },
11526            child_workflow_type,
11527            result: decode_wire_avro_value(result, codec)?,
11528        }));
11529    }
11530
11531    Ok(outcomes)
11532}
11533
11534fn payload_string(payload: &Value, key: &str) -> Option<String> {
11535    payload
11536        .get(key)
11537        .and_then(Value::as_str)
11538        .filter(|value| !value.is_empty())
11539        .map(str::to_string)
11540}
11541
11542fn workflow_failure_command(error: &Error) -> Value {
11543    let (exception_type, exception_class, properties) = match error {
11544        Error::ActivityFailed(failure) => (
11545            match failure.kind {
11546                ActivityFailureKind::Failed => "ActivityFailed",
11547                ActivityFailureKind::Cancelled => "ActivityCancelled",
11548                ActivityFailureKind::TimedOut => "ActivityTimedOut",
11549            },
11550            "durable_workflow::ActivityFailure",
11551            json!({
11552                "reason": failure.reason,
11553                "activity_execution_id": failure.activity_execution_id,
11554                "activity_attempt_id": failure.activity_attempt_id,
11555                "activity_type": failure.activity_type,
11556                "activity_class": failure.activity_class,
11557                "attempt_number": failure.attempt_number,
11558                "failure_id": failure.failure_id,
11559                "failure_category": failure.failure_category,
11560                "timeout_kind": failure.timeout_kind,
11561                "activity_non_retryable": failure.non_retryable,
11562                "activity_exception_type": failure.exception_type,
11563                "activity_exception_class": failure.exception_class,
11564                "activity_code": failure.code,
11565                "activity_exception": failure.exception,
11566            }),
11567        ),
11568        Error::ChildWorkflowFailed(failure) => (
11569            match failure.kind {
11570                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
11571                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
11572                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
11573            },
11574            "durable_workflow::ChildWorkflowFailure",
11575            json!({
11576                "reason": failure.reason,
11577                "parent_workflow_id": failure.parent_workflow_id,
11578                "parent_workflow_run_id": failure.parent_workflow_run_id,
11579                "child_workflow_id": failure.child_workflow_id,
11580                "child_workflow_run_id": failure.child_workflow_run_id,
11581                "child_workflow_type": failure.child_workflow_type,
11582                "failure_id": failure.failure_id,
11583                "failure_category": failure.failure_category,
11584                "child_exception_type": failure.exception_type,
11585                "child_exception_class": failure.exception_class,
11586                "child_non_retryable": failure.non_retryable,
11587                "child_code": failure.code,
11588                "child_exception": failure.exception,
11589            }),
11590        ),
11591        Error::ParallelFailed(failure) => (
11592            "ParallelFailed",
11593            "durable_workflow::ParallelFailure",
11594            json!({
11595                "parallel_group_id": failure.group_id,
11596                "parallel_member_path": failure.member_path,
11597                "parallel_group_path": failure.group_path,
11598                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
11599                "cause_type": workflow_error_type(&failure.cause),
11600                "cause_message": failure.cause.to_string(),
11601            }),
11602        ),
11603        Error::SagaCompensationFailed(failure) => (
11604            "SagaCompensationFailed",
11605            "durable_workflow::SagaCompensationFailure",
11606            json!({
11607                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
11608                "initiating_failure_message": failure.initiating_failure.to_string(),
11609                "compensation_activity_type": failure.compensation_activity_type,
11610                "compensation_registration_order": failure.compensation_registration_order,
11611                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
11612                "compensation_failure_message": failure.compensation_failure.to_string(),
11613            }),
11614        ),
11615        Error::WorkflowCancellationRequested(_) => (
11616            "WorkflowCancellationRequested",
11617            "durable_workflow::WorkflowCancellationRequested",
11618            json!({"reason": "cancelled"}),
11619        ),
11620        Error::NonDeterministicReplay(_) => (
11621            "NonDeterministicReplay",
11622            "durable_workflow::Error",
11623            Value::Null,
11624        ),
11625        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
11626    };
11627    let non_retryable = match error {
11628        Error::ActivityFailed(failure) => failure.non_retryable,
11629        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
11630        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
11631        Error::SagaCompensationFailed(failure) => {
11632            workflow_error_non_retryable(&failure.compensation_failure)
11633        }
11634        Error::WorkflowCancellationRequested(_) => true,
11635        Error::NonDeterministicReplay(_) => true,
11636        _ => false,
11637    };
11638
11639    json!({
11640        "type": "fail_workflow",
11641        "message": error.to_string(),
11642        "exception_type": exception_type,
11643        "exception_class": exception_class,
11644        "non_retryable": non_retryable,
11645        "exception": {
11646            "type": exception_type,
11647            "class": exception_class,
11648            "message": error.to_string(),
11649            "properties": properties,
11650        }
11651    })
11652}
11653
11654fn workflow_error_type(error: &Error) -> &'static str {
11655    match error {
11656        Error::ActivityFailed(failure) => match failure.kind {
11657            ActivityFailureKind::Failed => "ActivityFailed",
11658            ActivityFailureKind::Cancelled => "ActivityCancelled",
11659            ActivityFailureKind::TimedOut => "ActivityTimedOut",
11660        },
11661        Error::ChildWorkflowFailed(failure) => match failure.kind {
11662            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
11663            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
11664            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
11665        },
11666        Error::ParallelFailed(_) => "ParallelFailed",
11667        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
11668        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
11669        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
11670        _ => "RustWorkflowError",
11671    }
11672}
11673
11674fn workflow_error_non_retryable(error: &Error) -> bool {
11675    match error {
11676        Error::ActivityFailed(failure) => failure.non_retryable,
11677        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
11678        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
11679        Error::SagaCompensationFailed(failure) => {
11680            workflow_error_non_retryable(&failure.compensation_failure)
11681        }
11682        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
11683        _ => false,
11684    }
11685}
11686
11687fn workflow_task_integrity_error(error: &Error) -> bool {
11688    matches!(
11689        error,
11690        Error::NonDeterministicReplay(_)
11691            | Error::Protocol(_)
11692            | Error::MissingWorkflowCommandIdentity
11693            | Error::WorkflowStatePoisoned
11694    )
11695}
11696
11697fn decode_signal_event_arguments(
11698    event: &HistoryEvent,
11699    fallback_codec: &str,
11700) -> Result<Vec<AvroValue>> {
11701    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
11702    validate_payload_codec(codec)?;
11703    let raw = signal_history_payload(&event.payload);
11704    let decoded = match raw.filter(|value| !value.is_null()) {
11705        Some(value) => decode_wire_avro_value(value, codec)?,
11706        None => AvroValue::Array(Vec::new()),
11707    };
11708    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
11709        unreachable!("normalize_avro_arguments always returns an array");
11710    };
11711    Ok(arguments)
11712}
11713
11714fn decode_update_event_arguments(
11715    event: &HistoryEvent,
11716    fallback_codec: &str,
11717) -> Result<Vec<AvroValue>> {
11718    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
11719    validate_payload_codec(codec)?;
11720    let decoded = match event
11721        .payload
11722        .get("arguments")
11723        .filter(|value| !value.is_null())
11724    {
11725        Some(value) => decode_wire_avro_value(value, codec)?,
11726        None => AvroValue::Array(Vec::new()),
11727    };
11728    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
11729        unreachable!("normalize_avro_arguments always returns an array");
11730    };
11731    Ok(arguments)
11732}
11733
11734fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
11735    let Some(export_events) = task
11736        .history_export
11737        .as_ref()
11738        .and_then(|export| export.get("history_events"))
11739        .and_then(Value::as_array)
11740    else {
11741        return Ok(());
11742    };
11743
11744    if export_events.len() > task.history_events.len() {
11745        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
11746    }
11747
11748    Ok(())
11749}
11750
11751fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
11752    let Some(export) = task.history_export.as_ref() else {
11753        return Ok(());
11754    };
11755    let signals = export
11756        .get("signals")
11757        .and_then(Value::as_array)
11758        .cloned()
11759        .unwrap_or_default();
11760    let activities = export
11761        .get("activities")
11762        .and_then(Value::as_array)
11763        .cloned()
11764        .unwrap_or_default();
11765    let export_codec = export
11766        .get("payloads")
11767        .and_then(|payloads| payloads.get("codec"))
11768        .and_then(Value::as_str)
11769        .unwrap_or(&task.payload_codec)
11770        .to_string();
11771    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
11772
11773    for event in &mut task.history_events {
11774        if event.event_type == "ActivityCompleted" {
11775            let sequence = event
11776                .payload
11777                .get("sequence")
11778                .or_else(|| event.payload.get("workflow_sequence"))
11779                .and_then(value_as_u64);
11780            let Some(activity) = sequence.and_then(|sequence| {
11781                activities.iter().find(|activity| {
11782                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
11783                })
11784            }) else {
11785                continue;
11786            };
11787            let Some(payload) = event.payload.as_object_mut() else {
11788                continue;
11789            };
11790            if missing_payload(payload.get("result")) {
11791                if let Some(result) = activity
11792                    .get("result")
11793                    .filter(|value| !missing_payload(Some(value)))
11794                {
11795                    payload.insert("result".to_string(), result.clone());
11796                }
11797            }
11798            for field in ["payload_codec", "activity_type"] {
11799                if payload
11800                    .get(field)
11801                    .and_then(Value::as_str)
11802                    .unwrap_or_default()
11803                    .is_empty()
11804                {
11805                    if let Some(value) = activity.get(field) {
11806                        payload.insert(field.to_string(), value.clone());
11807                    }
11808                }
11809            }
11810            continue;
11811        }
11812
11813        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
11814            continue;
11815        }
11816        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
11817        let command_id = event
11818            .payload
11819            .get("workflow_command_id")
11820            .or_else(|| event.raw.get("workflow_command_id"))
11821            .and_then(Value::as_str);
11822        let signal_name = event
11823            .payload
11824            .get("signal_name")
11825            .and_then(Value::as_str)
11826            .unwrap_or_default()
11827            .to_string();
11828        let matched = signals
11829            .iter()
11830            .find(|signal| {
11831                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
11832            })
11833            .or_else(|| {
11834                signals.iter().find(|signal| {
11835                    command_id.is_some()
11836                        && signal.get("command_id").and_then(Value::as_str) == command_id
11837                })
11838            })
11839            .or_else(|| {
11840                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
11841                let signal = signals
11842                    .iter()
11843                    .filter(|signal| {
11844                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
11845                    })
11846                    .nth(*offset);
11847                if signal.is_some() {
11848                    *offset += 1;
11849                }
11850                signal
11851            });
11852        let Some(signal) = matched else {
11853            continue;
11854        };
11855        let signal_codec = signal
11856            .get("payload_codec")
11857            .and_then(Value::as_str)
11858            .unwrap_or(&export_codec);
11859        let Some(payload) = event.payload.as_object_mut() else {
11860            continue;
11861        };
11862        if missing_payload(payload.get("arguments")) {
11863            if let Some(arguments) = signal
11864                .get("arguments")
11865                .filter(|value| !missing_payload(Some(value)))
11866            {
11867                let envelope = match arguments {
11868                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
11869                    other => other.clone(),
11870                };
11871                payload.insert("arguments".to_string(), envelope);
11872            }
11873        }
11874        if payload
11875            .get("payload_codec")
11876            .and_then(Value::as_str)
11877            .unwrap_or_default()
11878            .is_empty()
11879        {
11880            payload.insert("payload_codec".to_string(), json!(signal_codec));
11881        }
11882    }
11883
11884    Ok(())
11885}
11886
11887fn missing_payload(value: Option<&Value>) -> bool {
11888    match value {
11889        None | Some(Value::Null) => true,
11890        Some(Value::String(value)) => value.is_empty(),
11891        Some(_) => false,
11892    }
11893}
11894
11895fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
11896    let export_signals = task
11897        .history_export
11898        .as_ref()
11899        .and_then(|export| export.get("signals"))
11900        .and_then(Value::as_array)
11901        .cloned()
11902        .unwrap_or_default();
11903    let export_codec = task
11904        .history_export
11905        .as_ref()
11906        .and_then(|export| export.get("payloads"))
11907        .and_then(|payloads| payloads.get("codec"))
11908        .and_then(Value::as_str)
11909        .unwrap_or(&task.payload_codec);
11910    let mut name_offsets: HashMap<String, usize> = HashMap::new();
11911    let mut signals = Vec::new();
11912
11913    for event in &task.history_events {
11914        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
11915            continue;
11916        }
11917
11918        let name = event
11919            .payload
11920            .get("signal_name")
11921            .and_then(Value::as_str)
11922            .unwrap_or_default();
11923        if name.is_empty() {
11924            continue;
11925        }
11926        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
11927        let command_id = event
11928            .payload
11929            .get("workflow_command_id")
11930            .or_else(|| event.raw.get("workflow_command_id"))
11931            .and_then(Value::as_str);
11932        let matched_export = export_signals
11933            .iter()
11934            .find(|candidate| {
11935                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
11936            })
11937            .or_else(|| {
11938                export_signals.iter().find(|candidate| {
11939                    command_id.is_some()
11940                        && candidate.get("command_id").and_then(Value::as_str) == command_id
11941                })
11942            })
11943            .or_else(|| {
11944                let offset = name_offsets.entry(name.to_string()).or_default();
11945                let candidate = export_signals
11946                    .iter()
11947                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
11948                    .nth(*offset);
11949                if candidate.is_some() {
11950                    *offset += 1;
11951                }
11952                candidate
11953            });
11954        let codec = event
11955            .payload
11956            .get("payload_codec")
11957            .and_then(Value::as_str)
11958            .or_else(|| {
11959                matched_export
11960                    .and_then(|signal| signal.get("payload_codec"))
11961                    .and_then(Value::as_str)
11962            })
11963            .unwrap_or(export_codec);
11964        let raw_arguments = signal_history_payload(&event.payload)
11965            .filter(|value| !value.is_null())
11966            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
11967        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
11968        let workflow_sequence = event
11969            .payload
11970            .get("workflow_sequence")
11971            .and_then(value_as_u64)
11972            .or_else(|| {
11973                matched_export
11974                    .and_then(|signal| signal.get("workflow_sequence"))
11975                    .and_then(value_as_u64)
11976            });
11977
11978        signals.push(QuerySignal {
11979            id: signal_id.map(str::to_string).or_else(|| {
11980                matched_export
11981                    .and_then(|signal| signal.get("id"))
11982                    .and_then(Value::as_str)
11983                    .map(str::to_string)
11984            }),
11985            name: name.to_string(),
11986            arguments,
11987            avro_arguments,
11988            workflow_sequence,
11989        });
11990    }
11991
11992    if signals.is_empty() {
11993        for signal in export_signals {
11994            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
11995                continue;
11996            }
11997            let Some(name) = signal.get("name").and_then(Value::as_str) else {
11998                continue;
11999            };
12000            let codec = signal
12001                .get("payload_codec")
12002                .and_then(Value::as_str)
12003                .unwrap_or(export_codec);
12004            let (arguments, avro_arguments) =
12005                decode_query_signal_arguments(signal.get("arguments"), codec)?;
12006            signals.push(QuerySignal {
12007                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
12008                name: name.to_string(),
12009                arguments,
12010                avro_arguments,
12011                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
12012            });
12013        }
12014        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
12015    }
12016
12017    Ok(signals)
12018}
12019
12020fn decode_query_signal_arguments(
12021    raw: Option<&Value>,
12022    codec: &str,
12023) -> Result<(Vec<Value>, Vec<AvroValue>)> {
12024    validate_payload_codec(codec)?;
12025    let decoded = match raw.filter(|value| !value.is_null()) {
12026        Some(value) => decode_wire_avro_value(value, codec)?,
12027        None => AvroValue::Array(Vec::new()),
12028    };
12029    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
12030        unreachable!("normalize_avro_arguments always returns an array");
12031    };
12032    let arguments = avro_arguments
12033        .iter()
12034        .cloned()
12035        .map(AvroValue::into_json)
12036        .collect::<Result<Vec<_>>>()?;
12037    Ok((arguments, avro_arguments))
12038}
12039
12040fn value_as_u64(value: &Value) -> Option<u64> {
12041    value
12042        .as_u64()
12043        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
12044}
12045
12046#[cfg(test)]
12047mod tests {
12048    use super::*;
12049    use std::{
12050        io::{Read, Write},
12051        net::{SocketAddr, TcpListener, TcpStream},
12052        sync::atomic::AtomicUsize,
12053        thread,
12054    };
12055
12056    #[derive(Clone, Copy, Debug)]
12057    enum InvalidTaskPayloadCodec {
12058        Missing,
12059        Null,
12060        NonString,
12061    }
12062
12063    impl InvalidTaskPayloadCodec {
12064        fn label(self) -> &'static str {
12065            match self {
12066                Self::Missing => "missing",
12067                Self::Null => "null",
12068                Self::NonString => "non-string",
12069            }
12070        }
12071
12072        fn apply(self, task: &mut Value) {
12073            let task = task.as_object_mut().expect("task fixture object");
12074            match self {
12075                Self::Missing => {
12076                    task.remove("payload_codec");
12077                }
12078                Self::Null => {
12079                    task.insert("payload_codec".to_string(), Value::Null);
12080                }
12081                Self::NonString => {
12082                    task.insert("payload_codec".to_string(), json!(42));
12083                }
12084            }
12085        }
12086    }
12087
12088    fn fixture_envelope(value: Value) -> Value {
12089        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
12090    }
12091
12092    fn fixture_blob(value: Value) -> String {
12093        encode_payload(&value, DEFAULT_CODEC)
12094            .expect("encode Avro test fixture")
12095            .blob
12096    }
12097
12098    #[test]
12099    fn client_builder_rejects_the_sdk_owned_api_suffix() {
12100        for base_url in [
12101            "http://127.0.0.1:8080/api",
12102            "http://localhost:8080/api/",
12103            "https://runtime.example.test/namespaces/orders/api",
12104        ] {
12105            let error = Client::builder(base_url)
12106                .build()
12107                .expect_err("SDK-owned /api suffix must be rejected during build");
12108
12109            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
12110            assert!(
12111                error.to_string().contains("SDK appends /api automatically"),
12112                "the validation error must explain how to fix the endpoint"
12113            );
12114        }
12115    }
12116
12117    #[test]
12118    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
12119        for (base_url, expected) in [
12120            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
12121            (
12122                "http://localhost:8080/durable-workflow/",
12123                "http://localhost:8080/durable-workflow",
12124            ),
12125            (
12126                "https://runtime.example.test/namespaces/orders",
12127                "https://runtime.example.test/namespaces/orders",
12128            ),
12129            (
12130                "https://runtime.example.test/gateway/api/namespaces/orders",
12131                "https://runtime.example.test/gateway/api/namespaces/orders",
12132            ),
12133            (
12134                "https://api.example.test/runtime/orders/",
12135                "https://api.example.test/runtime/orders",
12136            ),
12137        ] {
12138            let client = Client::builder(base_url)
12139                .build()
12140                .expect("Server and Cloud runtime base URL must remain valid");
12141
12142            assert_eq!(client.base_url, expected);
12143        }
12144    }
12145
12146    #[test]
12147    fn workflow_completion_uses_the_additive_command_protocol_floor() {
12148        assert_eq!(
12149            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
12150            WORKER_PROTOCOL_VERSION
12151        );
12152        assert_eq!(
12153            workflow_completion_protocol_version(&[json!({
12154                "type": "upsert_search_attributes",
12155                "attributes": {"OrderStatus": "waiting"},
12156            })]),
12157            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
12158        );
12159        assert_eq!(
12160            workflow_completion_protocol_version(&[
12161                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
12162                json!({"type": "open_condition_wait", "condition_key": "ready"}),
12163            ]),
12164            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
12165        );
12166    }
12167
12168    fn typed_fidelity_probe() -> AvroValue {
12169        AvroValue::Map(BTreeMap::from([
12170            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
12171            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
12172            (
12173                "numeric".to_string(),
12174                AvroValue::Map(BTreeMap::from([
12175                    ("0".to_string(), AvroValue::String("zero".to_string())),
12176                    ("1".to_string(), AvroValue::String("one".to_string())),
12177                ])),
12178            ),
12179            (
12180                "nested".to_string(),
12181                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
12182                    "enabled".to_string(),
12183                    AvroValue::Boolean(true),
12184                )]))]),
12185            ),
12186            (
12187                "projection_collisions".to_string(),
12188                AvroValue::Array(projection_collision_probe()),
12189            ),
12190        ]))
12191    }
12192
12193    fn projection_collision_probe() -> Vec<AvroValue> {
12194        vec![
12195            AvroValue::Map(BTreeMap::from([
12196                ("$type".to_string(), AvroValue::String("bytes".to_string())),
12197                (
12198                    "base64".to_string(),
12199                    AvroValue::String("ordinary user text".to_string()),
12200                ),
12201            ])),
12202            AvroValue::Map(BTreeMap::from([
12203                ("$type".to_string(), AvroValue::String("map".to_string())),
12204                (
12205                    "entries".to_string(),
12206                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
12207                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
12208                        (
12209                            "value".to_string(),
12210                            AvroValue::String("user map".to_string()),
12211                        ),
12212                    ]))]),
12213                ),
12214            ])),
12215        ]
12216    }
12217
12218    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12219    struct TypedContract {
12220        nested: TypedNested,
12221        mode: TypedMode,
12222        optional: Option<String>,
12223        absent: Option<String>,
12224        items: Vec<i64>,
12225        labels: BTreeMap<String, String>,
12226        bytes: serde_bytes::ByteBuf,
12227        signed: i64,
12228        finite: f64,
12229    }
12230
12231    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12232    struct TypedNested {
12233        enabled: bool,
12234    }
12235
12236    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12237    enum TypedMode {
12238        Detailed { label: String },
12239    }
12240
12241    fn typed_contract() -> TypedContract {
12242        TypedContract {
12243            nested: TypedNested { enabled: true },
12244            mode: TypedMode::Detailed {
12245                label: "compiler-checked".to_string(),
12246            },
12247            optional: Some("present".to_string()),
12248            absent: None,
12249            items: vec![i64::MIN, 0, i64::MAX],
12250            labels: BTreeMap::from([
12251                ("language".to_string(), "rust".to_string()),
12252                ("wire".to_string(), "avro".to_string()),
12253            ]),
12254            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
12255            signed: -9_223_372_036_854_775_000,
12256            finite: 12.5,
12257        }
12258    }
12259
12260    #[derive(Clone, Debug, Default, PartialEq)]
12261    struct ReplayCounterState {
12262        loaded: Option<String>,
12263        count: i64,
12264        finished: bool,
12265    }
12266
12267    fn replay_counter_worker() -> Worker {
12268        let client = Client::new("http://127.0.0.1:8080").expect("client");
12269        let mut worker = Worker::new(client, "rust-workers");
12270        worker.register_replayed_workflow(
12271            "replay-counter",
12272            ReplayCounterState::default,
12273            |ctx, _input, state| async move {
12274                let loaded = ctx.activity("load-counter", json!([])).await?;
12275                state.update(|current| {
12276                    current.loaded = loaded.as_str().map(str::to_string);
12277                })?;
12278                for _ in 0..2 {
12279                    let signal = ctx.wait_signal("increment").await?;
12280                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
12281                    state.update(|current| current.count += amount)?;
12282                }
12283                state.update(|current| current.finished = true)?;
12284                state.read(|current| Ok(json!(current.count)))?
12285            },
12286        );
12287        worker.register_replayed_query::<ReplayCounterState, _, _>(
12288            "replay-counter",
12289            "current",
12290            |_ctx, state, _args| async move {
12291                Ok(json!({
12292                    "loaded": state.loaded,
12293                    "count": state.count,
12294                    "finished": state.finished,
12295                }))
12296            },
12297        );
12298        worker.register_replayed_query::<ReplayCounterState, _, _>(
12299            "replay-counter",
12300            "detached-mutation",
12301            |_ctx, state, _args| async move {
12302                let mut detached = (*state).clone();
12303                detached.count = 999;
12304                Ok(json!(detached.count))
12305            },
12306        );
12307        worker.register_replayed_query::<ReplayCounterState, _, _>(
12308            "replay-counter",
12309            "failed-mutation",
12310            |_ctx, state, _args| async move {
12311                let mut detached = (*state).clone();
12312                detached.count = 999;
12313                Err(Error::WorkerLoop("query refused".to_string()))
12314            },
12315        );
12316        worker
12317    }
12318
12319    fn replay_counter_query(
12320        query_name: &str,
12321        history_events: Value,
12322        run_status: &str,
12323    ) -> QueryTask {
12324        let arguments = fixture_envelope(json!([]));
12325        serde_json::from_value(json!({
12326            "query_task_id": format!("query-{query_name}"),
12327            "workflow_type": "replay-counter",
12328            "query_name": query_name,
12329            "payload_codec": DEFAULT_CODEC,
12330            "workflow_arguments": arguments.clone(),
12331            "query_arguments": arguments,
12332            "history_events": history_events,
12333            "run_status": run_status,
12334        }))
12335        .expect("query task")
12336    }
12337
12338    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
12339        workflow_context_with_codec(history, DEFAULT_CODEC)
12340    }
12341
12342    fn workflow_context_with_codec(
12343        history: Vec<HistoryEvent>,
12344        payload_codec: &str,
12345    ) -> WorkflowContext {
12346        WorkflowContext {
12347            state: Arc::new(Mutex::new(
12348                WorkflowState::new_with_identity(
12349                    history,
12350                    None,
12351                    None,
12352                    "rust-workers".to_string(),
12353                    payload_codec.to_string(),
12354                    None,
12355                )
12356                .expect("valid workflow history"),
12357            )),
12358        }
12359    }
12360
12361    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
12362        HistoryEvent {
12363            event_type: event_type.to_string(),
12364            payload,
12365            raw: HashMap::new(),
12366        }
12367    }
12368
12369    fn parallel_path_entry(
12370        kind: &str,
12371        base: u64,
12372        size: usize,
12373        index: usize,
12374    ) -> ParallelGroupMetadata {
12375        parallel_group_entry(base, size, index, kind)
12376    }
12377
12378    fn parallel_history_event(
12379        event_type: &str,
12380        sequence: u64,
12381        identity_field: &str,
12382        identity: &str,
12383        path: Vec<ParallelGroupMetadata>,
12384        result: Option<Value>,
12385    ) -> HistoryEvent {
12386        let mut payload = serde_json::Map::from_iter([
12387            ("sequence".to_string(), json!(sequence)),
12388            (identity_field.to_string(), json!(identity)),
12389        ]);
12390        let inner = path.last().expect("parallel history path");
12391        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
12392        payload.insert("parallel_group_path".to_string(), json!(path));
12393        if let Some(result) = result {
12394            let field = if event_type == "ChildRunCompleted" {
12395                "result"
12396            } else {
12397                "result"
12398            };
12399            payload.insert(field.to_string(), fixture_envelope(result));
12400            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
12401        }
12402        history_event(event_type, Value::Object(payload))
12403    }
12404
12405    fn nested_parallel_operations() -> Vec<ParallelOperation> {
12406        vec![
12407            ParallelOperation::activity("first", json!([])),
12408            ParallelOperation::group(vec![
12409                ParallelOperation::child_workflow(
12410                    "second",
12411                    ChildWorkflowOptions::new("child-workers"),
12412                    json!([]),
12413                ),
12414                ParallelOperation::activity("third", json!([])),
12415            ]),
12416        ]
12417    }
12418
12419    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
12420        let outer = [
12421            parallel_path_entry("mixed", 1, 3, 0),
12422            parallel_path_entry("mixed", 1, 3, 1),
12423            parallel_path_entry("mixed", 1, 3, 2),
12424        ];
12425        [
12426            vec![outer[0].clone()],
12427            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
12428            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
12429        ]
12430    }
12431
12432    #[test]
12433    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
12434        let ctx = workflow_context(Vec::new());
12435        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
12436        let mut task_context = TaskContext::from_waker(noop_waker_ref());
12437
12438        assert!(matches!(
12439            call.as_mut().poll(&mut task_context),
12440            Poll::Pending
12441        ));
12442        let commands = ctx.take_commands().expect("parallel commands");
12443        assert_eq!(
12444            commands
12445                .iter()
12446                .map(|command| command["type"].as_str().unwrap_or_default())
12447                .collect::<Vec<_>>(),
12448            [
12449                "schedule_activity",
12450                "start_child_workflow",
12451                "schedule_activity"
12452            ]
12453        );
12454        let paths = nested_parallel_paths();
12455        for (command, path) in commands.iter().zip(paths) {
12456            assert_eq!(command["parallel_group_path"], json!(path));
12457            assert_eq!(
12458                command["parallel_group_id"],
12459                json!(path.last().expect("inner group").parallel_group_id)
12460            );
12461        }
12462    }
12463
12464    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
12465        let paths = nested_parallel_paths();
12466        let third = parallel_history_event(
12467            "ActivityCompleted",
12468            3,
12469            "activity_type",
12470            "third",
12471            paths[2].clone(),
12472            Some(json!("three")),
12473        );
12474        vec![
12475            parallel_history_event(
12476                "ActivityCompleted",
12477                1,
12478                "activity_type",
12479                "first",
12480                paths[0].clone(),
12481                Some(json!("one")),
12482            ),
12483            parallel_history_event(
12484                "ChildWorkflowScheduled",
12485                2,
12486                "child_workflow_type",
12487                "second",
12488                paths[1].clone(),
12489                None,
12490            ),
12491            parallel_history_event(
12492                "ChildRunCompleted",
12493                2,
12494                "child_workflow_type",
12495                "second",
12496                paths[1].clone(),
12497                Some(json!("two")),
12498            ),
12499            third.clone(),
12500            third,
12501        ]
12502    }
12503
12504    #[test]
12505    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
12506        for _restart_or_completed_replay in 0..2 {
12507            let ctx = workflow_context(completed_nested_parallel_history());
12508            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
12509            let mut task_context = TaskContext::from_waker(noop_waker_ref());
12510            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
12511                panic!("completed nested parallel history must replay");
12512            };
12513            assert_eq!(
12514                results,
12515                vec![
12516                    ParallelResult::Activity(json!("one")),
12517                    ParallelResult::Group(vec![
12518                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
12519                            parent: WorkflowIdentity {
12520                                workflow_id: None,
12521                                run_id: None,
12522                            },
12523                            child: WorkflowIdentity {
12524                                workflow_id: None,
12525                                run_id: None,
12526                            },
12527                            child_workflow_type: Some("second".to_string()),
12528                            result: json!("two"),
12529                        }),
12530                        ParallelResult::Activity(json!("three")),
12531                    ]),
12532                ]
12533            );
12534            assert!(ctx.take_commands().expect("commands").is_empty());
12535            ctx.ensure_history_consumed().expect("history consumed");
12536        }
12537    }
12538
12539    #[test]
12540    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
12541        let paths = nested_parallel_paths();
12542        let history = vec![
12543            parallel_history_event(
12544                "ActivityCompleted",
12545                1,
12546                "activity_type",
12547                "first",
12548                paths[0].clone(),
12549                Some(json!("one")),
12550            ),
12551            parallel_history_event(
12552                "ChildWorkflowScheduled",
12553                2,
12554                "child_workflow_type",
12555                "second",
12556                paths[1].clone(),
12557                None,
12558            ),
12559            parallel_history_event(
12560                "ChildRunFailed",
12561                2,
12562                "child_workflow_type",
12563                "second",
12564                paths[1].clone(),
12565                None,
12566            ),
12567            parallel_history_event(
12568                "ActivityCompleted",
12569                3,
12570                "activity_type",
12571                "third",
12572                paths[2].clone(),
12573                Some(json!("late")),
12574            ),
12575        ];
12576        let ctx = workflow_context(history);
12577        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
12578        let mut task_context = TaskContext::from_waker(noop_waker_ref());
12579        let outcome = call.as_mut().poll(&mut task_context);
12580        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
12581            panic!("one failed child must return a typed partial failure: {outcome:?}");
12582        };
12583        assert_eq!(failure.member_path, [1, 0]);
12584        assert_eq!(failure.group_id, "parallel-calls:1:3");
12585        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
12586        assert_eq!(
12587            failure
12588                .completed
12589                .iter()
12590                .map(|completion| completion.member_path.clone())
12591                .collect::<Vec<_>>(),
12592            [vec![0], vec![1, 1]]
12593        );
12594    }
12595
12596    #[test]
12597    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
12598        let paths = nested_parallel_paths();
12599        let history = vec![
12600            parallel_history_event(
12601                "ActivityScheduled",
12602                1,
12603                "activity_type",
12604                "first",
12605                paths[0].clone(),
12606                None,
12607            ),
12608            parallel_history_event(
12609                "ChildWorkflowScheduled",
12610                2,
12611                "child_workflow_type",
12612                "second",
12613                paths[1].clone(),
12614                None,
12615            ),
12616            parallel_history_event(
12617                "ActivityScheduled",
12618                3,
12619                "activity_type",
12620                "third",
12621                paths[2].clone(),
12622                None,
12623            ),
12624        ];
12625        for _restart in 0..2 {
12626            let ctx = workflow_context(history.clone());
12627            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
12628            let mut task_context = TaskContext::from_waker(noop_waker_ref());
12629            let outcome = call.as_mut().poll(&mut task_context);
12630            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
12631            assert!(ctx.take_commands().expect("commands").is_empty());
12632        }
12633    }
12634
12635    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
12636        let mut saga = ctx.saga();
12637        let outcome = async {
12638            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
12639            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
12640            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
12641            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
12642            ctx.activity("trip.charge", json!([])).await?;
12643            Ok(json!({"status": "booked"}))
12644        }
12645        .await;
12646        saga.finish(outcome).await
12647    }
12648
12649    fn saga_activity(
12650        event_type: &str,
12651        sequence: u64,
12652        activity_type: &str,
12653        result: Option<Value>,
12654    ) -> HistoryEvent {
12655        let mut payload = json!({
12656            "sequence": sequence,
12657            "activity_type": activity_type,
12658            "message": format!("{activity_type} failed"),
12659            "exception_type": "PlannedFailure",
12660            "non_retryable": true,
12661        });
12662        if let Some(result) = result {
12663            payload["result"] = fixture_envelope(result);
12664        }
12665        history_event(event_type, payload)
12666    }
12667
12668    #[test]
12669    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
12670        let completed_hotel_compensation = saga_activity(
12671            "ActivityCompleted",
12672            4,
12673            "trip.cancel-hotel",
12674            Some(Value::Null),
12675        );
12676        let history = vec![
12677            saga_activity(
12678                "ActivityCompleted",
12679                1,
12680                "trip.reserve-flight",
12681                Some(json!("flight-1")),
12682            ),
12683            saga_activity(
12684                "ActivityCompleted",
12685                2,
12686                "trip.reserve-hotel",
12687                Some(json!("hotel-1")),
12688            ),
12689            saga_activity("ActivityFailed", 3, "trip.charge", None),
12690            completed_hotel_compensation.clone(),
12691            completed_hotel_compensation,
12692        ];
12693
12694        for _restart in 0..2 {
12695            let ctx = workflow_context(history.clone());
12696            let mut future = Box::pin(trip_saga(ctx.clone()));
12697            let mut task_context = TaskContext::from_waker(noop_waker_ref());
12698            assert!(matches!(
12699                future.as_mut().poll(&mut task_context),
12700                Poll::Pending
12701            ));
12702            let commands = ctx.take_commands().expect("compensation command");
12703            assert_eq!(commands.len(), 1);
12704            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
12705        }
12706    }
12707
12708    #[test]
12709    fn saga_compensation_failure_preserves_both_typed_failures() {
12710        let history = vec![
12711            saga_activity(
12712                "ActivityCompleted",
12713                1,
12714                "trip.reserve-flight",
12715                Some(json!("flight-1")),
12716            ),
12717            saga_activity(
12718                "ActivityCompleted",
12719                2,
12720                "trip.reserve-hotel",
12721                Some(json!("hotel-1")),
12722            ),
12723            saga_activity("ActivityFailed", 3, "trip.charge", None),
12724            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
12725        ];
12726        let ctx = workflow_context(history);
12727        let mut future = Box::pin(trip_saga(ctx));
12728        let mut task_context = TaskContext::from_waker(noop_waker_ref());
12729        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
12730            future.as_mut().poll(&mut task_context)
12731        else {
12732            panic!("compensation failure must remain structured");
12733        };
12734        assert!(matches!(
12735            *failure.initiating_failure,
12736            Error::ActivityFailed(_)
12737        ));
12738        assert!(matches!(
12739            *failure.compensation_failure,
12740            Error::ActivityFailed(_)
12741        ));
12742        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
12743        assert_eq!(failure.compensation_registration_order, 2);
12744    }
12745
12746    #[test]
12747    fn saga_compensates_cooperative_cancellation() {
12748        let ctx = workflow_context(vec![saga_activity(
12749            "ActivityCompleted",
12750            1,
12751            "trip.reserve-flight",
12752            Some(json!("flight-1")),
12753        )]);
12754        ctx.state.lock().expect("state").cancel_requested = true;
12755        let run = {
12756            let ctx = ctx.clone();
12757            async move {
12758                let mut saga = ctx.saga();
12759                let outcome = async {
12760                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
12761                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
12762                    ctx.throw_if_cancellation_requested()?;
12763                    Ok(json!("unexpected"))
12764                }
12765                .await;
12766                saga.finish(outcome).await
12767            }
12768        };
12769        let mut future = Box::pin(run);
12770        let mut task_context = TaskContext::from_waker(noop_waker_ref());
12771        assert!(matches!(
12772            future.as_mut().poll(&mut task_context),
12773            Poll::Pending
12774        ));
12775        let commands = ctx.take_commands().expect("cancellation compensation");
12776        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
12777    }
12778
12779    fn workflow_task(
12780        workflow_type: &str,
12781        history_events: Vec<HistoryEvent>,
12782        payload_codec: &str,
12783    ) -> WorkflowTask {
12784        WorkflowTask {
12785            task_id: format!("wft-{workflow_type}"),
12786            workflow_command_id: None,
12787            workflow_id: Some(format!("wf-{workflow_type}")),
12788            run_id: Some(format!("run-{workflow_type}")),
12789            workflow_type: workflow_type.to_string(),
12790            cancel_requested: false,
12791            payload_codec: payload_codec.to_string(),
12792            arguments: Some(
12793                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
12794            ),
12795            total_history_events: Some(history_events.len() as u64),
12796            history_size_bytes: None,
12797            continue_as_new_recommended: None,
12798            history_budget_pressure: None,
12799            history_events,
12800            next_history_page_token: None,
12801            workflow_task_attempt: 1,
12802            workflow_signal_id: None,
12803            signal_name: None,
12804            signal_arguments: None,
12805            workflow_update_id: None,
12806            update_name: None,
12807            lease_owner: Some("rust-worker".to_string()),
12808        }
12809    }
12810
12811    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
12812    struct SideEffectProbe {
12813        request_id: String,
12814        attempt: u32,
12815    }
12816
12817    #[test]
12818    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
12819        let calls = AtomicUsize::new(0);
12820        let ctx = workflow_context(Vec::new());
12821        let value = ctx
12822            .side_effect(|| {
12823                calls.fetch_add(1, Ordering::SeqCst);
12824                SideEffectProbe {
12825                    request_id: "request-42".to_string(),
12826                    attempt: 3,
12827                }
12828            })
12829            .expect("first side effect");
12830        assert_eq!(value.attempt, 3);
12831        assert_eq!(calls.load(Ordering::SeqCst), 1);
12832        let commands = ctx.take_commands().expect("commands");
12833        assert_eq!(commands.len(), 1);
12834        assert_eq!(commands[0]["type"], "record_side_effect");
12835        assert_eq!(
12836            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
12837            serde_json::to_value(&value).expect("value")
12838        );
12839
12840        let replay = workflow_context(vec![history_event(
12841            "SideEffectRecorded",
12842            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
12843        )]);
12844        let replayed: SideEffectProbe = replay
12845            .side_effect(|| {
12846                calls.fetch_add(1, Ordering::SeqCst);
12847                panic!("committed side-effect callbacks must not run during replay")
12848            })
12849            .expect("replayed side effect");
12850        assert_eq!(replayed, value);
12851        assert_eq!(calls.load(Ordering::SeqCst), 1);
12852        assert!(replay.take_commands().expect("commands").is_empty());
12853        replay.ensure_history_consumed().expect("history consumed");
12854    }
12855
12856    #[test]
12857    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
12858        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
12859        let value = ctx
12860            .side_effect(|| SideEffectProbe {
12861                request_id: "avro-request".to_string(),
12862                attempt: 1,
12863            })
12864            .expect("Avro side effect");
12865        let uuid = ctx.uuid_v4().expect("deterministic UUID");
12866        let commands = ctx.take_commands().expect("commands");
12867        assert_eq!(commands.len(), 2);
12868        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
12869        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
12870        assert_eq!(
12871            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
12872            serde_json::to_value(&value).expect("value")
12873        );
12874
12875        let replay = workflow_context_with_codec(
12876            vec![
12877                history_event(
12878                    "SideEffectRecorded",
12879                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
12880                ),
12881                history_event(
12882                    "SideEffectRecorded",
12883                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
12884                ),
12885            ],
12886            DEFAULT_CODEC,
12887        );
12888        let replayed: SideEffectProbe = replay
12889            .side_effect(|| panic!("Avro callback must not run"))
12890            .expect("replayed Avro value");
12891        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
12892        assert_eq!(replayed, value);
12893        assert_eq!(replayed_uuid, uuid);
12894        assert!(replay.take_commands().expect("commands").is_empty());
12895    }
12896
12897    #[test]
12898    fn typed_side_effect_replay_preserves_bytes_and_maps() {
12899        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
12900        let value = ctx
12901            .side_effect_avro_value(typed_fidelity_probe)
12902            .expect("typed side effect");
12903        let commands = ctx.take_commands().expect("side-effect command");
12904        assert_eq!(
12905            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
12906                .expect("recorded side effect"),
12907            value
12908        );
12909
12910        let replay = workflow_context_with_codec(
12911            vec![history_event(
12912                "SideEffectRecorded",
12913                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
12914            )],
12915            DEFAULT_CODEC,
12916        );
12917        assert_eq!(
12918            replay
12919                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
12920                .expect("replayed typed side effect"),
12921            value
12922        );
12923    }
12924
12925    #[test]
12926    fn ordered_side_effects_share_the_durable_command_stream() {
12927        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
12928        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
12929        let ctx = workflow_context(vec![
12930            history_event(
12931                "SideEffectRecorded",
12932                json!({"sequence": 1, "result": first}),
12933            ),
12934            history_event(
12935                "SideEffectRecorded",
12936                json!({"sequence": 2, "result": second}),
12937            ),
12938        ]);
12939        let first: String = ctx
12940            .side_effect(|| panic!("first callback must not run"))
12941            .expect("first replay");
12942        let second: i32 = ctx
12943            .side_effect(|| panic!("second callback must not run"))
12944            .expect("second replay");
12945        assert_eq!(first, "first");
12946        assert_eq!(second, 29);
12947        ctx.ensure_history_consumed().expect("ordered history");
12948
12949        let reordered = workflow_context(vec![history_event(
12950            "VersionMarkerRecorded",
12951            json!({
12952                "sequence": 1,
12953                "change_id": "before-side-effect",
12954                "version": 1,
12955                "min_supported": 1,
12956                "max_supported": 1,
12957            }),
12958        )]);
12959        let error = reordered
12960            .side_effect(|| "new".to_string())
12961            .expect_err("command reordering must fail");
12962        assert!(matches!(
12963            error,
12964            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
12965                if reason == "recorded_command_mismatch"
12966        ));
12967    }
12968
12969    #[test]
12970    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
12971        let ctx = workflow_context(Vec::new());
12972        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
12973        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
12974        assert!(ctx.patched("new-search").expect("patch"));
12975        ctx.deprecate_patch("new-search").expect("deprecate patch");
12976        let commands = ctx.take_commands().expect("commands");
12977        assert_eq!(commands.len(), 2);
12978        assert_eq!(commands[0]["type"], "record_version_marker");
12979        assert_eq!(commands[0]["version"], 2);
12980        assert_eq!(commands[1]["change_id"], "new-search");
12981
12982        let replay = workflow_context(vec![history_event(
12983            "VersionMarkerRecorded",
12984            json!({
12985                "sequence": 1,
12986                "change_id": "checkout-v2",
12987                "version": 2,
12988                "min_supported": 1,
12989                "max_supported": 2,
12990            }),
12991        )]);
12992        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
12993        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
12994        assert!(replay.take_commands().expect("commands").is_empty());
12995        replay.ensure_history_consumed().expect("history consumed");
12996    }
12997
12998    #[test]
12999    fn version_markers_reject_incompatible_or_malformed_history() {
13000        let incompatible = workflow_context(vec![history_event(
13001            "VersionMarkerRecorded",
13002            json!({
13003                "sequence": 1,
13004                "change_id": "checkout-v2",
13005                "version": 1,
13006                "min_supported": 1,
13007                "max_supported": 2,
13008            }),
13009        )]);
13010        let error = incompatible
13011            .get_version("checkout-v2", 2, 3)
13012            .expect_err("old version is unsupported");
13013        assert!(matches!(
13014            error,
13015            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13016                if reason == "version_marker_incompatible_range"
13017        ));
13018
13019        for (history, reason) in [
13020            (
13021                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
13022                "side_effect_result_missing",
13023            ),
13024            (
13025                vec![history_event(
13026                    "SideEffectRecorded",
13027                    json!({
13028                        "sequence": 1,
13029                        "result": {"codec": "avro", "blob": "not-base64"},
13030                    }),
13031                )],
13032                "side_effect_payload_incompatible",
13033            ),
13034            (
13035                vec![history_event(
13036                    "SideEffectRecorded",
13037                    json!({"sequence": 1, "result": {"unwrapped": true}}),
13038                )],
13039                "side_effect_payload_malformed",
13040            ),
13041            (
13042                vec![history_event(
13043                    "VersionMarkerRecorded",
13044                    json!({
13045                        "sequence": 1,
13046                        "change_id": "change",
13047                        "version": 1,
13048                        "min_supported": 2,
13049                        "max_supported": 1,
13050                    }),
13051                )],
13052                "version_marker_history_range_invalid",
13053            ),
13054        ] {
13055            let error = WorkflowState::new(
13056                history,
13057                "rust-workers".to_string(),
13058                DEFAULT_CODEC.to_string(),
13059                None,
13060            )
13061            .expect_err("malformed history must fail");
13062            assert!(matches!(
13063                error,
13064                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
13065                    if actual == reason
13066            ));
13067        }
13068    }
13069
13070    #[test]
13071    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
13072        let history = vec![history_event(
13073            "SearchAttributesUpserted",
13074            json!({
13075                "sequence": 1,
13076                "attributes": {"customer_tier": "gold"},
13077                "attribute_types": {"customer_tier": "keyword"},
13078                "merged": {"customer_tier": "gold"}
13079            }),
13080        )];
13081
13082        let matching = workflow_context(history.clone());
13083        matching
13084            .upsert_search_attributes(
13085                SearchAttributeUpdate::new()
13086                    .keyword("customer_tier", "gold")
13087                    .expect("keyword update"),
13088            )
13089            .expect("matching typed update must replay");
13090        matching
13091            .ensure_history_consumed()
13092            .expect("history consumed");
13093
13094        let changed_type = workflow_context(history.clone());
13095        let error = changed_type
13096            .upsert_search_attributes(
13097                SearchAttributeUpdate::new()
13098                    .string("customer_tier", "gold")
13099                    .expect("string update"),
13100            )
13101            .expect_err("same JSON value with a different declaration must be nondeterministic");
13102        let Error::NonDeterministicReplay(failure) = error else {
13103            panic!("typed identity drift must be a replay failure");
13104        };
13105        assert_eq!(failure.reason, "search_attribute_type_mismatch");
13106        assert_eq!(failure.sequence, Some(1));
13107
13108        let changed_value = workflow_context(history);
13109        let error = changed_value
13110            .upsert_search_attributes(
13111                SearchAttributeUpdate::new()
13112                    .keyword("customer_tier", "platinum")
13113                    .expect("keyword update"),
13114            )
13115            .expect_err("changed values must be nondeterministic");
13116        let Error::NonDeterministicReplay(failure) = error else {
13117            panic!("value drift must be a replay failure");
13118        };
13119        assert_eq!(failure.reason, "search_attribute_value_mismatch");
13120    }
13121
13122    #[test]
13123    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
13124        let history = vec![history_event(
13125            "SearchAttributesUpserted",
13126            json!({
13127                "sequence": 1,
13128                "attributes": {"customer_tier": "gold"},
13129                "merged": {"customer_tier": "gold"}
13130            }),
13131        )];
13132
13133        for update in [
13134            SearchAttributeUpdate::new()
13135                .keyword("customer_tier", "gold")
13136                .expect("keyword update"),
13137            SearchAttributeUpdate::new()
13138                .string("customer_tier", "gold")
13139                .expect("string update"),
13140        ] {
13141            let restarted = workflow_context(history.clone());
13142            restarted
13143                .upsert_search_attributes(update)
13144                .expect("legacy history constrains values but has unknown type identity");
13145            restarted
13146                .ensure_history_consumed()
13147                .expect("history consumed");
13148        }
13149    }
13150
13151    #[test]
13152    fn search_attribute_command_emits_canonical_types() {
13153        let ctx = workflow_context(Vec::new());
13154        ctx.upsert_search_attributes(
13155            SearchAttributeUpdate::new()
13156                .keyword("customer_tier", "gold")
13157                .expect("keyword update")
13158                .int("attempts", 3)
13159                .expect("int update")
13160                .delete("obsolete")
13161                .expect("delete update"),
13162        )
13163        .expect("valid search attributes");
13164
13165        assert_eq!(
13166            ctx.take_commands().expect("commands"),
13167            vec![json!({
13168                "type": "upsert_search_attributes",
13169                "attributes": {
13170                    "attempts": 3,
13171                    "customer_tier": "gold",
13172                    "obsolete": null
13173                },
13174                "attribute_types": {
13175                    "attempts": "int",
13176                    "customer_tier": "keyword"
13177                }
13178            })]
13179        );
13180    }
13181
13182    #[test]
13183    fn duplicate_side_effects_and_version_markers_are_rejected() {
13184        let duplicate_side_effect = WorkflowState::new(
13185            vec![
13186                history_event(
13187                    "SideEffectRecorded",
13188                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
13189                ),
13190                history_event(
13191                    "SideEffectRecorded",
13192                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
13193                ),
13194            ],
13195            "rust-workers".to_string(),
13196            DEFAULT_CODEC.to_string(),
13197            None,
13198        )
13199        .expect_err("duplicate side effect");
13200        assert!(matches!(
13201            duplicate_side_effect,
13202            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13203                if reason == "duplicate_side_effect_record"
13204        ));
13205
13206        let marker = |sequence| {
13207            history_event(
13208                "VersionMarkerRecorded",
13209                json!({
13210                    "sequence": sequence,
13211                    "change_id": "same-change",
13212                    "version": 1,
13213                    "min_supported": 1,
13214                    "max_supported": 1,
13215                }),
13216            )
13217        };
13218        let duplicate_marker = WorkflowState::new(
13219            vec![marker(1), marker(3)],
13220            "rust-workers".to_string(),
13221            DEFAULT_CODEC.to_string(),
13222            None,
13223        )
13224        .expect_err("duplicate marker");
13225        assert!(matches!(
13226            duplicate_marker,
13227            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13228                if reason == "duplicate_version_marker"
13229        ));
13230    }
13231
13232    #[test]
13233    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
13234        let mut state = WorkflowState::new(
13235            Vec::new(),
13236            "rust-workers".to_string(),
13237            DEFAULT_CODEC.to_string(),
13238            None,
13239        )
13240        .expect("workflow state");
13241        state.workflow_command_identity = "command-7".to_string();
13242        let context = WorkflowContext {
13243            state: Arc::new(Mutex::new(state)),
13244        };
13245        let item =
13246            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
13247
13248        context
13249            .append_workflow_stream("output", &[item], Some(10))
13250            .expect("append command");
13251        context
13252            .error_workflow_stream("output", "producer failed", None)
13253            .expect("error command");
13254        let commands = context.take_commands().expect("commands");
13255
13256        assert_eq!(commands[0]["type"], "record_side_effect");
13257        assert_eq!(
13258            commands[0]["workflow_stream"]["command_identity"],
13259            "command-7"
13260        );
13261        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
13262        assert_eq!(
13263            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
13264            "dw-stream:command-7:0:0"
13265        );
13266        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
13267
13268        let recorded = history_event(
13269            "SideEffectRecorded",
13270            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
13271        );
13272        let mut replay_state = WorkflowState::new(
13273            vec![recorded],
13274            "rust-workers".to_string(),
13275            DEFAULT_CODEC.to_string(),
13276            None,
13277        )
13278        .expect("replay state");
13279        replay_state.workflow_command_identity = "command-7".to_string();
13280        let replay_context = WorkflowContext {
13281            state: Arc::new(Mutex::new(replay_state)),
13282        };
13283        replay_context
13284            .append_workflow_stream(
13285                "output",
13286                &[WorkflowStreamAppendItem::from_reference(
13287                    "s3://bucket/item.avro",
13288                )],
13289                Some(10),
13290            )
13291            .expect("replayed append");
13292        assert!(replay_context
13293            .take_commands()
13294            .expect("replayed commands")
13295            .is_empty());
13296    }
13297
13298    #[test]
13299    fn workflow_stream_authoring_requires_server_durable_command_identity() {
13300        let context = workflow_context(Vec::new());
13301        let error = context
13302            .append_workflow_stream(
13303                "output",
13304                &[WorkflowStreamAppendItem::from_reference(
13305                    "s3://bucket/item.avro",
13306                )],
13307                None,
13308            )
13309            .expect_err("stream append without durable command identity must fail closed");
13310
13311        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
13312        assert!(context.take_commands().expect("commands").is_empty());
13313    }
13314
13315    #[test]
13316    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
13317        fn worker(calls: Arc<AtomicUsize>) -> Worker {
13318            let client = Client::new("http://127.0.0.1:8080").expect("client");
13319            let mut worker = Worker::new(client, "rust-workers");
13320            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
13321                let calls = Arc::clone(&calls);
13322                async move {
13323                    let captured = ctx.side_effect(|| {
13324                        calls.fetch_add(1, Ordering::SeqCst);
13325                        "captured-once".to_string()
13326                    })?;
13327                    let version = ctx.get_version("cold-restart", 1, 2)?;
13328                    Ok(json!({"captured": captured, "version": version}))
13329                }
13330            });
13331            worker
13332        }
13333
13334        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
13335            WorkflowTask {
13336                task_id: "wft-side-effect-version".to_string(),
13337                workflow_command_id: None,
13338                workflow_id: Some("wf-side-effect-version".to_string()),
13339                run_id: Some("run-side-effect-version".to_string()),
13340                workflow_type: "rust.side-effect-version".to_string(),
13341                cancel_requested: false,
13342                payload_codec: DEFAULT_CODEC.to_string(),
13343                arguments: Some(
13344                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
13345                ),
13346                history_events,
13347                total_history_events: None,
13348                history_size_bytes: None,
13349                continue_as_new_recommended: None,
13350                history_budget_pressure: None,
13351                next_history_page_token: None,
13352                workflow_task_attempt: 1,
13353                workflow_signal_id: None,
13354                signal_name: None,
13355                signal_arguments: None,
13356                workflow_update_id: None,
13357                update_name: None,
13358                lease_owner: Some("rust-worker".to_string()),
13359            }
13360        }
13361
13362        let calls = Arc::new(AtomicUsize::new(0));
13363        let initial = worker(Arc::clone(&calls))
13364            .execute_workflow_task(task(Vec::new()))
13365            .expect("initial execution");
13366        assert_eq!(
13367            initial
13368                .iter()
13369                .map(|command| &command["type"])
13370                .collect::<Vec<_>>(),
13371            vec![
13372                "record_side_effect",
13373                "record_version_marker",
13374                "complete_workflow"
13375            ]
13376        );
13377        assert_eq!(calls.load(Ordering::SeqCst), 1);
13378
13379        let restarted = worker(Arc::clone(&calls));
13380        let replayed = restarted
13381            .execute_workflow_task(task(vec![
13382                history_event(
13383                    "SideEffectRecorded",
13384                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
13385                ),
13386                history_event(
13387                    "VersionMarkerRecorded",
13388                    json!({
13389                        "sequence": 2,
13390                        "change_id": "cold-restart",
13391                        "version": 2,
13392                        "min_supported": 1,
13393                        "max_supported": 2,
13394                    }),
13395                ),
13396            ]))
13397            .expect("cold replay");
13398        assert_eq!(replayed.len(), 1);
13399        assert_eq!(replayed[0]["type"], "complete_workflow");
13400        assert_eq!(calls.load(Ordering::SeqCst), 1);
13401    }
13402
13403    #[test]
13404    fn side_effect_replay_rejects_changed_rust_value_type() {
13405        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
13406        let ctx = workflow_context(vec![history_event(
13407            "SideEffectRecorded",
13408            json!({"sequence": 1, "result": result}),
13409        )]);
13410        let error = ctx
13411            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
13412            .expect_err("changed type must fail replay");
13413        assert!(matches!(
13414            error,
13415            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
13416                if reason == "side_effect_type_mismatch"
13417        ));
13418    }
13419
13420    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
13421        vec![
13422            history_event(
13423                "ActivityScheduled",
13424                json!({
13425                    "sequence": 1,
13426                    "activity_type": "flaky",
13427                    "activity_execution_id": "act-1",
13428                    "activity": {
13429                        "id": "act-1",
13430                        "sequence": 1,
13431                        "type": "flaky",
13432                        "queue": "critical-activities",
13433                        "execution_mode": null,
13434                        "retry_policy": {
13435                            "snapshot_version": 1,
13436                            "max_attempts": 3,
13437                            "backoff_seconds": [2, 4],
13438                            "start_to_close_timeout": 30,
13439                            "schedule_to_start_timeout": 5,
13440                            "schedule_to_close_timeout": 90,
13441                            "heartbeat_timeout": 10,
13442                            "non_retryable_error_types": ["PermanentError"]
13443                        }
13444                    }
13445                }),
13446            ),
13447            history_event(
13448                "ActivityStarted",
13449                json!({
13450                    "sequence": 1,
13451                    "activity_type": "flaky",
13452                    "activity_execution_id": "act-1",
13453                    "activity_attempt_id": "attempt-1",
13454                    "attempt_number": 1
13455                }),
13456            ),
13457            history_event(
13458                "ActivityRetryScheduled",
13459                json!({
13460                    "sequence": 1,
13461                    "activity_type": "flaky",
13462                    "activity_execution_id": "act-1",
13463                    "activity_attempt_id": "attempt-1",
13464                    "attempt_number": 1,
13465                    "retry_after_attempt": 1,
13466                    "retry_backoff_seconds": 2,
13467                    "failure_category": "activity",
13468                    "exception_type": "TransientError"
13469                }),
13470            ),
13471            history_event(
13472                "ActivityStarted",
13473                json!({
13474                    "sequence": 1,
13475                    "activity_type": "flaky",
13476                    "activity_execution_id": "act-1",
13477                    "activity_attempt_id": "attempt-2",
13478                    "attempt_number": 2
13479                }),
13480            ),
13481            history_event(
13482                "ActivityCompleted",
13483                json!({
13484                    "sequence": 1,
13485                    "activity_type": "flaky",
13486                    "activity_execution_id": "act-1",
13487                    "activity_attempt_id": "attempt-2",
13488                    "attempt_number": 2,
13489                    "payload_codec": DEFAULT_CODEC,
13490                    "result": fixture_envelope(json!({"status":"recovered"}))
13491                }),
13492            ),
13493        ]
13494    }
13495
13496    fn retry_activity_options() -> ActivityOptions {
13497        ActivityOptions::new()
13498            .task_queue("critical-activities")
13499            .retry_policy(
13500                ActivityRetryPolicy::new(3)
13501                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
13502                    .non_retryable_error_type("PermanentError"),
13503            )
13504            .start_to_close_timeout(Duration::from_secs(30))
13505            .schedule_to_start_timeout(Duration::from_secs(5))
13506            .schedule_to_close_timeout(Duration::from_secs(90))
13507            .heartbeat_timeout(Duration::from_secs(10))
13508    }
13509
13510    #[test]
13511    fn fixed_avro_value_round_trips_json_values() {
13512        let value = json!({"greeting": "hello", "count": 3, "ok": true});
13513        let envelope = PayloadEnvelope::avro(&value).expect("encode");
13514        assert_eq!(envelope.codec, DEFAULT_CODEC);
13515        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
13516    }
13517
13518    #[tokio::test]
13519    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
13520        let client = Client::new("http://127.0.0.1:8080").expect("client");
13521        let mut worker = Worker::new(client, "rust-workers");
13522        worker.register_typed_workflow(
13523            "typed.contract.workflow",
13524            |_ctx, input: TypedContract| async move { Ok(input) },
13525        );
13526        worker.register_typed_activity(
13527            "typed.contract.activity",
13528            |_ctx, input: TypedContract| async move { Ok(input) },
13529        );
13530
13531        let expected = typed_contract();
13532        let arguments = AvroValue::Array(vec![
13533            AvroValue::from_serialize(&expected).expect("typed request")
13534        ]);
13535        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
13536        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
13537        workflow.arguments = Some(envelope.clone());
13538        let commands = worker
13539            .execute_workflow_task(workflow)
13540            .expect("typed workflow task");
13541        let workflow_result: TypedContract =
13542            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13543                .expect("workflow result envelope")
13544                .deserialize()
13545                .expect("workflow result type");
13546        assert_eq!(workflow_result, expected);
13547
13548        let activity = ActivityTask {
13549            task_id: "typed-contract-activity".to_string(),
13550            activity_attempt_id: Some("typed-contract-attempt".to_string()),
13551            attempt_id: None,
13552            activity_type: "typed.contract.activity".to_string(),
13553            payload_codec: DEFAULT_CODEC.to_string(),
13554            arguments: Some(envelope),
13555            attempt_number: 1,
13556            lease_owner: Some("rust-worker".to_string()),
13557        };
13558        let activity_result: TypedContract = worker
13559            .execute_activity_task(activity)
13560            .await
13561            .expect("typed activity task")
13562            .deserialize()
13563            .expect("activity result type");
13564        assert_eq!(activity_result, expected);
13565    }
13566
13567    #[tokio::test]
13568    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
13569        let client = Client::new("http://127.0.0.1:8080").expect("client");
13570        let mut worker = Worker::new(client, "rust-workers");
13571        worker.register_typed_workflow(
13572            "typed.shape.workflow",
13573            |_ctx, input: TypedContract| async move { Ok(input) },
13574        );
13575        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
13576            Ok(f64::NAN)
13577        });
13578
13579        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
13580        workflow.arguments = Some(
13581            encode_typed_envelope(
13582                &AvroValue::Array(vec![
13583                    AvroValue::String("first".to_string()),
13584                    AvroValue::String("second".to_string()),
13585                ]),
13586                DEFAULT_CODEC,
13587            )
13588            .expect("malformed typed arguments"),
13589        );
13590        let commands = worker
13591            .execute_workflow_task(workflow)
13592            .expect("shape mismatch becomes a workflow failure");
13593        let message = commands[0]["message"].as_str().expect("failure message");
13594        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
13595        assert!(message.contains(type_name::<TypedContract>()));
13596        assert!(message.contains("task carried 2 arguments"));
13597
13598        let activity = ActivityTask {
13599            task_id: "typed-unsupported-activity".to_string(),
13600            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
13601            attempt_id: None,
13602            activity_type: "typed.unsupported.activity".to_string(),
13603            payload_codec: DEFAULT_CODEC.to_string(),
13604            arguments: Some(
13605                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
13606                    .expect("unit arguments"),
13607            ),
13608            attempt_number: 1,
13609            lease_owner: Some("rust-worker".to_string()),
13610        };
13611        let Error::HandlerType {
13612            handler_kind,
13613            handler_name,
13614            value_kind,
13615            rust_type,
13616            message,
13617        } = worker
13618            .execute_activity_task(activity)
13619            .await
13620            .expect_err("non-finite handler output must fail")
13621        else {
13622            panic!("expected contextual handler type failure");
13623        };
13624        assert_eq!(handler_kind, HandlerKind::Activity);
13625        assert_eq!(handler_name, "typed.unsupported.activity");
13626        assert_eq!(value_kind, HandlerValueKind::Result);
13627        assert_eq!(rust_type, type_name::<f64>());
13628        assert!(message.contains("non_finite_float"));
13629    }
13630
13631    #[tokio::test]
13632    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
13633        #[derive(Clone, Default)]
13634        struct State {
13635            observed: Option<TypedContract>,
13636        }
13637
13638        let client = Client::new("http://127.0.0.1:8080").expect("client");
13639        let mut worker = Worker::new(client, "rust-workers");
13640        worker.register_typed_replayed_workflow(
13641            "typed.contract.replayed",
13642            State::default,
13643            |ctx, input: TypedContract, state| async move {
13644                let result: TypedContract =
13645                    ctx.activity_typed("typed.contract.activity", input).await?;
13646                state.update(|current| current.observed = Some(result.clone()))?;
13647                Ok(result)
13648            },
13649        );
13650        worker.register_replayed_query::<State, _, _>(
13651            "typed.contract.replayed",
13652            "observed",
13653            |_ctx, state, _args| async move {
13654                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
13655            },
13656        );
13657
13658        let expected = typed_contract();
13659        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
13660        let workflow_arguments =
13661            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
13662                .expect("workflow arguments");
13663        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
13664        let task = QueryTask {
13665            query_task_id: "typed-replay-query".to_string(),
13666            query_task_attempt: 1,
13667            lease_owner: Some("rust-worker".to_string()),
13668            workflow_id: Some("typed-replay".to_string()),
13669            run_id: Some("typed-replay-run".to_string()),
13670            workflow_type: "typed.contract.replayed".to_string(),
13671            query_name: "observed".to_string(),
13672            payload_codec: DEFAULT_CODEC.to_string(),
13673            workflow_arguments: Some(workflow_arguments),
13674            query_arguments: Some(
13675                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
13676                    .expect("query arguments"),
13677            ),
13678            history_events: vec![
13679                history_event(
13680                    "ActivityScheduled",
13681                    json!({
13682                        "sequence": 1,
13683                        "activity_type": "typed.contract.activity"
13684                    }),
13685                ),
13686                history_event(
13687                    "ActivityCompleted",
13688                    json!({
13689                        "sequence": 1,
13690                        "activity_type": "typed.contract.activity",
13691                        "payload_codec": DEFAULT_CODEC,
13692                        "result": result
13693                    }),
13694                ),
13695            ],
13696            history_export: None,
13697            run_status: Some("completed".to_string()),
13698        };
13699
13700        assert_eq!(
13701            worker
13702                .execute_query_task(task)
13703                .await
13704                .expect("typed replay query")
13705                .deserialize::<i64>()
13706                .expect("query result"),
13707            expected.signed
13708        );
13709    }
13710
13711    #[tokio::test]
13712    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
13713        let client = Client::new("http://127.0.0.1:8080").expect("client");
13714        let mut worker = Worker::new(client, "rust-workers");
13715        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
13716        worker
13717            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
13718        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
13719            Ok(input)
13720        });
13721        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
13722            Ok(input)
13723        });
13724        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
13725            Ok(AvroValue::Array(
13726                ctx.wait_signal_avro_value("changed").await?,
13727            ))
13728        });
13729
13730        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
13731        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
13732
13733        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
13734        workflow.arguments = Some(envelope.clone());
13735        let commands = worker
13736            .execute_workflow_task(workflow)
13737            .expect("typed workflow task");
13738        assert_eq!(commands[0]["type"], "complete_workflow");
13739        assert_eq!(
13740            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13741                .expect("typed workflow result"),
13742            arguments
13743        );
13744
13745        let activity = ActivityTask {
13746            task_id: "activity-typed".to_string(),
13747            activity_attempt_id: Some("attempt-typed".to_string()),
13748            attempt_id: None,
13749            activity_type: "typed.activity".to_string(),
13750            payload_codec: DEFAULT_CODEC.to_string(),
13751            arguments: Some(envelope.clone()),
13752            attempt_number: 1,
13753            lease_owner: Some("rust-worker".to_string()),
13754        };
13755        assert_eq!(
13756            worker
13757                .execute_activity_task(activity)
13758                .await
13759                .expect("typed activity result"),
13760            arguments
13761        );
13762
13763        let query = QueryTask {
13764            query_task_id: "query-typed".to_string(),
13765            query_task_attempt: 1,
13766            lease_owner: Some("rust-worker".to_string()),
13767            workflow_id: Some("typed-1".to_string()),
13768            run_id: Some("run-typed".to_string()),
13769            workflow_type: "typed.echo".to_string(),
13770            query_name: "inspect".to_string(),
13771            payload_codec: DEFAULT_CODEC.to_string(),
13772            workflow_arguments: Some(
13773                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
13774                    .expect("workflow input"),
13775            ),
13776            query_arguments: Some(envelope.clone()),
13777            history_events: Vec::new(),
13778            history_export: None,
13779            run_status: Some("running".to_string()),
13780        };
13781        assert_eq!(
13782            worker
13783                .execute_query_task(query)
13784                .await
13785                .expect("typed query result"),
13786            arguments
13787        );
13788
13789        let mut update = workflow_task(
13790            "typed.echo",
13791            vec![history_event(
13792                "UpdateAccepted",
13793                json!({
13794                    "update_id": "update-typed",
13795                    "update_name": "replace",
13796                    "arguments": envelope.clone(),
13797                }),
13798            )],
13799            DEFAULT_CODEC,
13800        );
13801        update.workflow_update_id = Some("update-typed".to_string());
13802        update.update_name = Some("replace".to_string());
13803        let commands = worker
13804            .execute_workflow_task(update)
13805            .expect("typed update task");
13806        assert_eq!(commands[0]["type"], "complete_update");
13807        assert_eq!(
13808            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13809                .expect("typed update result"),
13810            arguments
13811        );
13812
13813        let mut signal = workflow_task(
13814            "typed.signal",
13815            vec![history_event(
13816                "SignalReceived",
13817                json!({
13818                    "signal_id": "signal-typed",
13819                    "signal_name": "changed",
13820                    "arguments": envelope.clone(),
13821                }),
13822            )],
13823            DEFAULT_CODEC,
13824        );
13825        signal.workflow_signal_id = Some("signal-typed".to_string());
13826        signal.signal_name = Some("changed".to_string());
13827        signal.signal_arguments = Some(envelope);
13828        let commands = worker
13829            .execute_workflow_task(signal)
13830            .expect("typed signal resume");
13831        assert_eq!(
13832            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
13833                .expect("typed signal result"),
13834            arguments
13835        );
13836    }
13837
13838    #[tokio::test]
13839    async fn typed_helpers_never_parse_json_inspection_projection() {
13840        let collision_values = projection_collision_probe();
13841        let expected = AvroValue::Array(collision_values.clone());
13842        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
13843
13844        let activity_context = workflow_context_with_codec(
13845            vec![history_event(
13846                "ActivityCompleted",
13847                json!({
13848                    "sequence": 1,
13849                    "activity_type": "collision.activity",
13850                    "payload_codec": DEFAULT_CODEC,
13851                    "result": envelope.clone(),
13852                }),
13853            )],
13854            DEFAULT_CODEC,
13855        );
13856        assert_eq!(
13857            activity_context
13858                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
13859                .await
13860                .expect("typed activity collision result"),
13861            expected
13862        );
13863
13864        let signal_context = workflow_context_with_codec(
13865            vec![
13866                history_event(
13867                    "SignalWaitOpened",
13868                    json!({"sequence": 1, "signal_name": "collision"}),
13869                ),
13870                history_event(
13871                    "SignalApplied",
13872                    json!({
13873                        "sequence": 1,
13874                        "signal_name": "collision",
13875                        "payload_codec": DEFAULT_CODEC,
13876                        "value": envelope.clone(),
13877                    }),
13878                ),
13879            ],
13880            DEFAULT_CODEC,
13881        );
13882        assert_eq!(
13883            signal_context
13884                .wait_signal_avro_value("collision")
13885                .await
13886                .expect("typed signal collision arguments"),
13887            collision_values
13888        );
13889
13890        let child_context = workflow_context_with_codec(
13891            vec![
13892                history_event(
13893                    "ChildWorkflowScheduled",
13894                    json!({
13895                        "sequence": 1,
13896                        "child_workflow_instance_id": "collision-child",
13897                        "child_workflow_run_id": "collision-run",
13898                        "child_workflow_type": "collision.child",
13899                    }),
13900                ),
13901                history_event(
13902                    "ChildRunCompleted",
13903                    json!({
13904                        "sequence": 1,
13905                        "child_workflow_instance_id": "collision-child",
13906                        "child_workflow_run_id": "collision-run",
13907                        "child_workflow_type": "collision.child",
13908                        "payload_codec": DEFAULT_CODEC,
13909                        "result": envelope,
13910                    }),
13911                ),
13912            ],
13913            DEFAULT_CODEC,
13914        );
13915        let child = child_context
13916            .start_child_workflow_avro_value(
13917                "collision.child",
13918                ChildWorkflowOptions::new("collision-workers"),
13919                AvroValue::Array(Vec::new()),
13920            )
13921            .await
13922            .expect("typed child collision result");
13923        assert_eq!(child.result, expected);
13924    }
13925
13926    #[tokio::test]
13927    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
13928        let client = Client::new("http://127.0.0.1:8080").expect("client");
13929        let mut worker = Worker::new(client, "rust-workers");
13930        worker.register_replayed_workflow_avro_value(
13931            "typed.replayed",
13932            || (),
13933            |_ctx, input, _state| async move { Ok(input) },
13934        );
13935        worker.register_replayed_query_avro_value::<(), _, _>(
13936            "typed.replayed",
13937            "inspect",
13938            |ctx, _state, args| async move {
13939                let mut signals = ctx.signals_avro_value("collision");
13940                let signal = signals
13941                    .pop()
13942                    .map(AvroValue::Array)
13943                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
13944                Ok(AvroValue::Array(vec![
13945                    ctx.workflow_input_avro_value().clone(),
13946                    signal,
13947                    args,
13948                ]))
13949            },
13950        );
13951        let arguments = AvroValue::Array(projection_collision_probe());
13952        let signal_arguments =
13953            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
13954        let task = QueryTask {
13955            query_task_id: "query-typed-replay".to_string(),
13956            query_task_attempt: 1,
13957            lease_owner: Some("rust-worker".to_string()),
13958            workflow_id: Some("typed-replay".to_string()),
13959            run_id: Some("run-typed-replay".to_string()),
13960            workflow_type: "typed.replayed".to_string(),
13961            query_name: "inspect".to_string(),
13962            payload_codec: DEFAULT_CODEC.to_string(),
13963            workflow_arguments: Some(
13964                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
13965            ),
13966            query_arguments: Some(
13967                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
13968            ),
13969            history_events: vec![history_event(
13970                "SignalReceived",
13971                json!({
13972                    "signal_id": "collision-signal",
13973                    "signal_name": "collision",
13974                    "workflow_sequence": 1,
13975                    "payload_codec": DEFAULT_CODEC,
13976                    "arguments": signal_arguments,
13977                }),
13978            )],
13979            history_export: None,
13980            run_status: Some("completed".to_string()),
13981        };
13982
13983        assert_eq!(
13984            worker
13985                .execute_query_task(task)
13986                .await
13987                .expect("typed replay query"),
13988            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
13989        );
13990    }
13991
13992    #[test]
13993    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
13994        let value = BTreeMap::from([(1_i32, "integer key")]);
13995        let error = PayloadEnvelope::avro(&value)
13996            .expect_err("integer map keys must fail")
13997            .to_string();
13998
13999        assert!(error.contains("invalid_map_key"));
14000    }
14001
14002    #[test]
14003    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
14004        let envelope = PayloadEnvelope {
14005            codec: "json".to_string(),
14006            blob: r#"{"greeting":"hello"}"#.to_string(),
14007        };
14008
14009        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
14010        let diagnostic = error.to_string();
14011        assert!(diagnostic.contains("unsupported_payload_codec"));
14012        assert!(diagnostic.contains("codec=\"avro\""));
14013        assert!(diagnostic.contains("HTTP document transport"));
14014    }
14015
14016    #[test]
14017    fn untagged_json_payload_value_fails_closed() {
14018        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
14019            .expect_err("untagged JSON payload values must fail");
14020        let diagnostic = error.to_string();
14021        assert!(diagnostic.contains("unsupported_payload_codec"));
14022        assert!(diagnostic.contains("untagged durable payload"));
14023        assert!(diagnostic.contains("HTTP document transport"));
14024    }
14025
14026    #[test]
14027    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
14028        let envelope = PayloadEnvelope {
14029            codec: DEFAULT_CODEC.to_string(),
14030            blob: BASE64.encode([0x01]),
14031        };
14032
14033        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
14034        assert!(error.to_string().contains("invalid_payload_framing"));
14035    }
14036
14037    #[tokio::test]
14038    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
14039        let server = MockWorkerServer::start();
14040        let client = Client::builder(server.base_url())
14041            .timeout(Duration::from_secs(2))
14042            .build()
14043            .expect("client");
14044        let invalid_commands = [
14045            json!({
14046                "type": "complete_workflow",
14047                "result": {"codec": "json", "blob": null}
14048            }),
14049            json!({
14050                "type": "schedule_activity",
14051                "arguments": {"codec": "yaml", "blob": "ignored"}
14052            }),
14053            json!({
14054                "type": "start_child_workflow",
14055                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
14056            }),
14057            json!({"type": "continue_as_new", "arguments": []}),
14058            json!({"type": "complete_update"}),
14059            json!({"type": "record_side_effect", "result": null}),
14060            json!({
14061                "type": "start_service_operation",
14062                "payload_codec": DEFAULT_CODEC,
14063                "request_payload": "raw-avro-bytes"
14064            }),
14065        ];
14066
14067        for command in invalid_commands {
14068            let error = client
14069                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
14070                .await
14071                .expect_err("invalid durable payload must fail locally");
14072            let diagnostic = error.to_string();
14073            assert!(
14074                diagnostic.contains("unsupported_payload_codec")
14075                    || diagnostic.contains("invalid_payload_envelope")
14076                    || diagnostic.contains("untagged durable payload"),
14077                "unexpected validation diagnostic: {diagnostic}"
14078            );
14079        }
14080
14081        assert_eq!(
14082            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
14083            0,
14084            "invalid command payloads must not reach HTTP transport"
14085        );
14086    }
14087
14088    #[test]
14089    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
14090        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
14091        let commands = [
14092            json!({"type": "complete_workflow", "result": envelope.clone()}),
14093            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
14094            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
14095            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
14096            json!({"type": "complete_update", "result": envelope.clone()}),
14097            json!({"type": "record_side_effect", "result": envelope.clone()}),
14098            json!({
14099                "type": "start_service_operation",
14100                "payload_codec": DEFAULT_CODEC,
14101                "request_payload": envelope.clone()
14102            }),
14103            json!({
14104                "type": "complete_workflow",
14105                "result": envelope,
14106                "metadata": {
14107                    "codec": "json",
14108                    "payload_codec": "customer-codec",
14109                    "result": {"codec": "yaml", "blob": null}
14110                }
14111            }),
14112        ];
14113
14114        validate_workflow_task_commands(&commands)
14115            .expect("customer metadata must not become a protocol codec declaration");
14116    }
14117
14118    #[test]
14119    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
14120        assert_eq!(
14121            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
14122            AvroValue::Array(Vec::new())
14123        );
14124        assert_eq!(
14125            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
14126            AvroValue::Array(Vec::new())
14127        );
14128
14129        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
14130        signal.signal_name = Some("empty-signal".to_string());
14131        signal.signal_arguments = None;
14132        let decoded = decode_resume_signal(&signal)
14133            .expect("valid Avro signal")
14134            .expect("named signal resumes the workflow");
14135        assert!(decoded.arguments.is_empty());
14136    }
14137
14138    #[tokio::test]
14139    async fn malformed_task_level_codecs_become_pre_handler_failures() {
14140        let client = Client::new("http://127.0.0.1:8080").expect("client");
14141        let mut worker = Worker::new(client, "rust-workers");
14142        let handler_calls = Arc::new(AtomicUsize::new(0));
14143
14144        let calls = Arc::clone(&handler_calls);
14145        worker.register_workflow("codec.workflow", move |_ctx, _args| {
14146            calls.fetch_add(1, Ordering::SeqCst);
14147            async move { Ok(Value::Null) }
14148        });
14149        let calls = Arc::clone(&handler_calls);
14150        worker.register_activity("codec.activity", move |_ctx, _args| {
14151            calls.fetch_add(1, Ordering::SeqCst);
14152            async move { Ok(Value::Null) }
14153        });
14154        let calls = Arc::clone(&handler_calls);
14155        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14156            calls.fetch_add(1, Ordering::SeqCst);
14157            async move { Ok(Value::Null) }
14158        });
14159
14160        let mut failures = Vec::new();
14161        for codec_case in [
14162            InvalidTaskPayloadCodec::Missing,
14163            InvalidTaskPayloadCodec::Null,
14164            InvalidTaskPayloadCodec::NonString,
14165        ] {
14166            let mut workflow = json!({
14167                "task_id": format!("workflow-{}", codec_case.label()),
14168                "workflow_type": "codec.workflow"
14169            });
14170            codec_case.apply(&mut workflow);
14171            match serde_json::from_value::<WorkflowTask>(workflow) {
14172                Ok(task) => match worker.execute_workflow_task(task) {
14173                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
14174                    outcome => failures.push(format!(
14175                        "workflow {} codec returned {outcome:?}",
14176                        codec_case.label()
14177                    )),
14178                },
14179                Err(error) => failures.push(format!(
14180                    "workflow {} codec failed transport deserialization: {error}",
14181                    codec_case.label()
14182                )),
14183            }
14184
14185            let mut activity = json!({
14186                "task_id": format!("activity-{}", codec_case.label()),
14187                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
14188                "activity_type": "codec.activity",
14189                "attempt_number": 1
14190            });
14191            codec_case.apply(&mut activity);
14192            match serde_json::from_value::<ActivityTask>(activity) {
14193                Ok(task) => match worker.execute_activity_task(task).await {
14194                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
14195                    outcome => failures.push(format!(
14196                        "activity {} codec returned {outcome:?}",
14197                        codec_case.label()
14198                    )),
14199                },
14200                Err(error) => failures.push(format!(
14201                    "activity {} codec failed transport deserialization: {error}",
14202                    codec_case.label()
14203                )),
14204            }
14205
14206            let mut query = json!({
14207                "query_task_id": format!("query-{}", codec_case.label()),
14208                "workflow_type": "codec.workflow",
14209                "query_name": "known"
14210            });
14211            codec_case.apply(&mut query);
14212            match serde_json::from_value::<QueryTask>(query) {
14213                Ok(task) => match worker.execute_query_task(task).await {
14214                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
14215                    outcome => failures.push(format!(
14216                        "query {} codec returned {outcome:?}",
14217                        codec_case.label()
14218                    )),
14219                },
14220                Err(error) => failures.push(format!(
14221                    "query {} codec failed transport deserialization: {error}",
14222                    codec_case.label()
14223                )),
14224            }
14225        }
14226
14227        assert!(failures.is_empty(), "{}", failures.join("\n"));
14228        assert_eq!(
14229            handler_calls.load(Ordering::SeqCst),
14230            0,
14231            "invalid task codecs must not invoke a handler"
14232        );
14233    }
14234
14235    #[tokio::test]
14236    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
14237        for codec_case in [
14238            InvalidTaskPayloadCodec::Missing,
14239            InvalidTaskPayloadCodec::Null,
14240            InvalidTaskPayloadCodec::NonString,
14241        ] {
14242            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
14243            let client = Client::builder(server.base_url())
14244                .timeout(Duration::from_secs(2))
14245                .build()
14246                .expect("client");
14247            let mut worker = Worker::new(client, "rust-workers")
14248                .worker_id("codec-worker")
14249                .poll_timeout(Duration::from_millis(10));
14250            let handler_calls = Arc::new(AtomicUsize::new(0));
14251
14252            let calls = Arc::clone(&handler_calls);
14253            worker.register_workflow("codec.workflow", move |_ctx, _args| {
14254                calls.fetch_add(1, Ordering::SeqCst);
14255                async move { Ok(Value::Null) }
14256            });
14257            let calls = Arc::clone(&handler_calls);
14258            worker.register_activity("codec.activity", move |_ctx, _args| {
14259                calls.fetch_add(1, Ordering::SeqCst);
14260                async move { Ok(Value::Null) }
14261            });
14262            let calls = Arc::clone(&handler_calls);
14263            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14264                calls.fetch_add(1, Ordering::SeqCst);
14265                async move { Ok(Value::Null) }
14266            });
14267
14268            assert_eq!(
14269                worker.run_once().await.expect("invalid tasks are settled"),
14270                3,
14271                "all {} codec tasks must be handled",
14272                codec_case.label()
14273            );
14274            assert_eq!(
14275                handler_calls.load(Ordering::SeqCst),
14276                0,
14277                "{} task codecs must fail before every handler",
14278                codec_case.label()
14279            );
14280
14281            for path in [
14282                "/api/worker/workflow-tasks/codec-workflow/fail",
14283                "/api/worker/activity-tasks/codec-activity/fail",
14284                "/api/worker/query-tasks/codec-query/fail",
14285            ] {
14286                let body = server.request_body(path);
14287                assert!(
14288                    body["failure"]["message"]
14289                        .as_str()
14290                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
14291                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
14292                    codec_case.label()
14293                );
14294            }
14295            assert_eq!(
14296                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
14297                    ["reason"],
14298                "query_payload_decode_failed"
14299            );
14300            for path in [
14301                "/api/worker/workflow-tasks/codec-workflow/complete",
14302                "/api/worker/activity-tasks/codec-activity/complete",
14303                "/api/worker/query-tasks/codec-query/complete",
14304            ] {
14305                assert_eq!(
14306                    server.request_count(path),
14307                    0,
14308                    "invalid {} codec task reached {path}",
14309                    codec_case.label()
14310                );
14311            }
14312        }
14313    }
14314
14315    #[tokio::test]
14316    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
14317        let client = Client::new("http://127.0.0.1:8080").expect("client");
14318        let mut worker = Worker::new(client, "rust-workers");
14319        let handler_calls = Arc::new(AtomicUsize::new(0));
14320
14321        let calls = Arc::clone(&handler_calls);
14322        worker.register_workflow("codec.workflow", move |_ctx, _args| {
14323            calls.fetch_add(1, Ordering::SeqCst);
14324            async move { Ok(Value::Null) }
14325        });
14326        let calls = Arc::clone(&handler_calls);
14327        worker.register_activity("codec.activity", move |_ctx, _args| {
14328            calls.fetch_add(1, Ordering::SeqCst);
14329            async move { Ok(Value::Null) }
14330        });
14331        let calls = Arc::clone(&handler_calls);
14332        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
14333            calls.fetch_add(1, Ordering::SeqCst);
14334            async move { Ok(Value::Null) }
14335        });
14336        let calls = Arc::clone(&handler_calls);
14337        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
14338            calls.fetch_add(1, Ordering::SeqCst);
14339            async move { Ok(Value::Null) }
14340        });
14341
14342        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
14343        workflow.payload_codec = "json".to_string();
14344        workflow.arguments = None;
14345        let error = worker
14346            .execute_workflow_task(workflow)
14347            .expect_err("task codec must be checked before workflow invocation");
14348        assert!(error.to_string().contains("unsupported_payload_codec"));
14349
14350        let activity = ActivityTask {
14351            task_id: "activity-invalid-codec".to_string(),
14352            activity_attempt_id: None,
14353            attempt_id: None,
14354            activity_type: "codec.activity".to_string(),
14355            payload_codec: "unknown".to_string(),
14356            arguments: None,
14357            attempt_number: 1,
14358            lease_owner: None,
14359        };
14360        let error = worker
14361            .execute_activity_task(activity)
14362            .await
14363            .expect_err("task codec must be checked before activity invocation");
14364        assert!(error.to_string().contains("unsupported_payload_codec"));
14365
14366        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
14367        update.workflow_update_id = Some("update-invalid-codec".to_string());
14368        update.update_name = Some("known".to_string());
14369        update.history_events.push(history_event(
14370            "UpdateAccepted",
14371            json!({
14372                "update_id": "update-invalid-codec",
14373                "update_name": "known",
14374                "arguments": {"codec": "json", "blob": null}
14375            }),
14376        ));
14377        let error = worker
14378            .execute_workflow_task(update)
14379            .expect_err("nested update codec must be checked before handler lookup");
14380        assert!(error.to_string().contains("unsupported_payload_codec"));
14381
14382        let query: QueryTask = serde_json::from_value(json!({
14383            "query_task_id": "query-invalid-codec",
14384            "workflow_type": "codec.workflow",
14385            "query_name": "known",
14386            "payload_codec": DEFAULT_CODEC,
14387            "workflow_arguments": null,
14388            "query_arguments": null,
14389            "history_export": {
14390                "payloads": {"codec": DEFAULT_CODEC},
14391                "signals": [{
14392                    "name": "empty",
14393                    "payload_codec": "json",
14394                    "arguments": null
14395                }]
14396            }
14397        }))
14398        .expect("query task");
14399        let failure = worker
14400            .execute_query_task(query)
14401            .await
14402            .expect_err("exported signal codec must be checked before query invocation");
14403        assert_eq!(failure.reason, "query_payload_decode_failed");
14404        assert!(failure.message.contains("unsupported_payload_codec"));
14405
14406        let exported_history: QueryTask = serde_json::from_value(json!({
14407            "query_task_id": "query-invalid-history-codec",
14408            "workflow_type": "codec.workflow",
14409            "query_name": "known",
14410            "payload_codec": DEFAULT_CODEC,
14411            "history_export": {
14412                "payloads": {"codec": DEFAULT_CODEC},
14413                "history_events": [{
14414                    "type": "ActivityCompleted",
14415                    "payload": {"payload_codec": "unknown", "result": null}
14416                }]
14417            }
14418        }))
14419        .expect("query task");
14420        let failure = worker
14421            .execute_query_task(exported_history)
14422            .await
14423            .expect_err("exported history codec must be checked before query invocation");
14424        assert_eq!(failure.reason, "query_payload_decode_failed");
14425        assert!(failure.message.contains("unsupported_payload_codec"));
14426        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
14427
14428        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
14429        unknown_workflow.arguments = None;
14430        unknown_workflow.history_events.push(history_event(
14431            "SignalReceived",
14432            json!({
14433                "signal_name": "empty",
14434                "payload_codec": "json",
14435                "arguments": null
14436            }),
14437        ));
14438        let error = worker
14439            .execute_workflow_task(unknown_workflow)
14440            .expect_err("history codec must precede unknown workflow outcome");
14441        assert!(error.to_string().contains("unsupported_payload_codec"));
14442
14443        let unknown_activity = ActivityTask {
14444            task_id: "activity-unknown".to_string(),
14445            activity_attempt_id: None,
14446            attempt_id: None,
14447            activity_type: "missing".to_string(),
14448            payload_codec: "json".to_string(),
14449            arguments: None,
14450            attempt_number: 1,
14451            lease_owner: None,
14452        };
14453        let error = worker
14454            .execute_activity_task(unknown_activity)
14455            .await
14456            .expect_err("codec must precede unknown activity outcome");
14457        assert!(error.to_string().contains("unsupported_payload_codec"));
14458
14459        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
14460        unknown_update.payload_codec = "json".to_string();
14461        unknown_update.arguments = None;
14462        unknown_update.workflow_update_id = Some("update-unknown".to_string());
14463        unknown_update.update_name = Some("missing".to_string());
14464        let error = worker
14465            .execute_workflow_task(unknown_update)
14466            .expect_err("codec must precede fail_update shortcut");
14467        assert!(error.to_string().contains("unsupported_payload_codec"));
14468
14469        let unknown_query: QueryTask = serde_json::from_value(json!({
14470            "query_task_id": "query-unknown",
14471            "workflow_type": "missing",
14472            "query_name": "missing",
14473            "payload_codec": "json",
14474            "workflow_arguments": null,
14475            "query_arguments": null
14476        }))
14477        .expect("query task");
14478        let failure = worker
14479            .execute_query_task(unknown_query)
14480            .await
14481            .expect_err("codec must precede unknown query outcome");
14482        assert_eq!(failure.reason, "query_payload_decode_failed");
14483        assert!(failure.message.contains("unsupported_payload_codec"));
14484    }
14485
14486    #[tokio::test]
14487    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
14488        let client = Client::new("http://127.0.0.1:8080").expect("client");
14489        let worker = Worker::new(client, "rust-workers");
14490
14491        for event_type in ["SignalReceived", "SignalApplied"] {
14492            for (payload_field, codec) in [
14493                ("value", "json"),
14494                ("input", "unknown"),
14495                ("arguments", "json"),
14496            ] {
14497                let payload = json!({
14498                    "signal_name": "empty",
14499                    payload_field: {"codec": codec, "blob": null}
14500                });
14501                let workflow = workflow_task(
14502                    "missing",
14503                    vec![history_event(event_type, payload.clone())],
14504                    DEFAULT_CODEC,
14505                );
14506                let error = worker
14507                    .execute_workflow_task(workflow)
14508                    .expect_err("signal payload codec must precede unknown workflow outcome");
14509                assert!(
14510                    error.to_string().contains("unsupported_payload_codec"),
14511                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
14512                );
14513
14514                let query: QueryTask = serde_json::from_value(json!({
14515                    "query_task_id": format!("query-{event_type}-{payload_field}"),
14516                    "workflow_type": "missing",
14517                    "query_name": "missing",
14518                    "payload_codec": DEFAULT_CODEC,
14519                    "workflow_arguments": null,
14520                    "query_arguments": null,
14521                    "history_events": [{
14522                        "event_type": event_type,
14523                        "payload": payload
14524                    }]
14525                }))
14526                .expect("query task");
14527                let failure = worker
14528                    .execute_query_task(query)
14529                    .await
14530                    .expect_err("signal payload codec must precede unknown query outcome");
14531                assert_eq!(
14532                    failure.reason, "query_payload_decode_failed",
14533                    "{event_type}.{payload_field} returned an unrelated query outcome"
14534                );
14535                assert!(
14536                    failure.message.contains("unsupported_payload_codec"),
14537                    "{event_type}.{payload_field} returned an unrelated query error: {}",
14538                    failure.message
14539                );
14540            }
14541        }
14542    }
14543
14544    #[test]
14545    fn workflow_context_schedules_activity_until_completion_is_in_history() {
14546        let ctx = WorkflowContext {
14547            state: Arc::new(Mutex::new(
14548                WorkflowState::new_with_identity(
14549                    Vec::new(),
14550                    Some("wf-parent".to_string()),
14551                    Some("run-parent".to_string()),
14552                    "rust-workers".to_string(),
14553                    DEFAULT_CODEC.to_string(),
14554                    None,
14555                )
14556                .expect("workflow state"),
14557            )),
14558        };
14559
14560        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
14561        let mut task_context = TaskContext::from_waker(noop_waker_ref());
14562        assert!(matches!(
14563            call.as_mut().poll(&mut task_context),
14564            Poll::Pending
14565        ));
14566
14567        let commands = ctx.take_commands().expect("commands");
14568        assert_eq!(commands[0]["type"], "schedule_activity");
14569        assert_eq!(commands[0]["activity_type"], "hello.activity");
14570    }
14571
14572    #[test]
14573    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
14574        let ctx = workflow_context(Vec::new());
14575        let options = ActivityOptions::new()
14576            .task_queue("payments")
14577            .retry_policy(
14578                ActivityRetryPolicy::new(4)
14579                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
14580                    .non_retryable_error_type("ValidationError"),
14581            )
14582            .start_to_close_timeout(Duration::from_secs(120))
14583            .schedule_to_start_timeout(Duration::from_secs(10))
14584            .schedule_to_close_timeout(Duration::from_secs(300))
14585            .heartbeat_timeout(Duration::from_secs(15));
14586        let mut call = Box::pin(ctx.activity_with_options(
14587            "charge-card",
14588            options,
14589            json!([{"order_id": "o-1"}]),
14590        ));
14591        let mut task_context = TaskContext::from_waker(noop_waker_ref());
14592
14593        assert!(matches!(
14594            call.as_mut().poll(&mut task_context),
14595            Poll::Pending
14596        ));
14597        assert!(matches!(
14598            call.as_mut().poll(&mut task_context),
14599            Poll::Pending
14600        ));
14601
14602        let commands = ctx.take_commands().expect("activity command");
14603        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
14604        assert_eq!(commands[0]["queue"], "payments");
14605        assert_eq!(
14606            commands[0]["retry_policy"],
14607            json!({
14608                "max_attempts": 4,
14609                "backoff_seconds": [1, 3, 9],
14610                "non_retryable_error_types": ["ValidationError"],
14611            })
14612        );
14613        assert_eq!(commands[0]["start_to_close_timeout"], 120);
14614        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
14615        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
14616        assert_eq!(commands[0]["heartbeat_timeout"], 15);
14617    }
14618
14619    #[test]
14620    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
14621        let ctx = workflow_context(Vec::new());
14622        let options = ActivityOptions::new().retry_policy(
14623            ActivityRetryPolicy::new(3)
14624                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
14625        );
14626        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
14627        let mut task_context = TaskContext::from_waker(noop_waker_ref());
14628
14629        assert!(matches!(
14630            call.as_mut().poll(&mut task_context),
14631            Poll::Pending
14632        ));
14633        assert_eq!(
14634            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
14635            json!([1, 2])
14636        );
14637    }
14638
14639    #[test]
14640    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
14641        let cases = [
14642            (
14643                ActivityOptions::new().task_queue("  "),
14644                ActivityOptionsErrorKind::EmptyTaskQueue,
14645            ),
14646            (
14647                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
14648                ActivityOptionsErrorKind::EmptyRetryPolicy,
14649            ),
14650            (
14651                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
14652                ActivityOptionsErrorKind::InvalidMaxAttempts,
14653            ),
14654            (
14655                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
14656                    max_attempts: None,
14657                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
14658                    non_retryable_error_types: Vec::new(),
14659                }),
14660                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
14661            ),
14662            (
14663                ActivityOptions::new().retry_policy(
14664                    ActivityRetryPolicy::new(2)
14665                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
14666                ),
14667                ActivityOptionsErrorKind::TooManyBackoffIntervals,
14668            ),
14669            (
14670                ActivityOptions::new().retry_policy(
14671                    ActivityRetryPolicy::new(2).exponential_backoff(
14672                        Duration::from_secs(1),
14673                        0,
14674                        None,
14675                    ),
14676                ),
14677                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
14678            ),
14679            (
14680                ActivityOptions::new()
14681                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
14682                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
14683            ),
14684            (
14685                ActivityOptions::new().retry_policy(
14686                    ActivityRetryPolicy::new(10_002).exponential_backoff(
14687                        Duration::from_secs(1),
14688                        1,
14689                        None,
14690                    ),
14691                ),
14692                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
14693            ),
14694            (
14695                ActivityOptions::new().retry_policy(
14696                    ActivityRetryPolicy::new(2)
14697                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
14698                ),
14699                ActivityOptionsErrorKind::BackoffOverflow,
14700            ),
14701        ];
14702
14703        for (options, expected_kind) in cases {
14704            let ctx = workflow_context(Vec::new());
14705            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
14706            let mut task_context = TaskContext::from_waker(noop_waker_ref());
14707            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
14708                call.as_mut().poll(&mut task_context)
14709            else {
14710                panic!("expected typed activity validation error");
14711            };
14712            assert_eq!(error.kind, expected_kind);
14713            assert!(ctx.take_commands().expect("commands").is_empty());
14714        }
14715    }
14716
14717    #[test]
14718    fn activity_options_validate_positive_and_ordered_timeouts() {
14719        let zero_timeout_cases = [
14720            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
14721            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
14722            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
14723            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
14724        ];
14725        for options in zero_timeout_cases {
14726            assert_eq!(
14727                options.validate().expect_err("zero timeout").kind,
14728                ActivityOptionsErrorKind::TimeoutNotPositive
14729            );
14730        }
14731
14732        let ordering_cases = [
14733            ActivityOptions::new()
14734                .heartbeat_timeout(Duration::from_secs(11))
14735                .start_to_close_timeout(Duration::from_secs(10)),
14736            ActivityOptions::new()
14737                .start_to_close_timeout(Duration::from_secs(31))
14738                .schedule_to_close_timeout(Duration::from_secs(30)),
14739            ActivityOptions::new()
14740                .schedule_to_start_timeout(Duration::from_secs(31))
14741                .schedule_to_close_timeout(Duration::from_secs(30)),
14742        ];
14743        for options in ordering_cases {
14744            assert_eq!(
14745                options.validate().expect_err("timeout order").kind,
14746                ActivityOptionsErrorKind::TimeoutOrder
14747            );
14748        }
14749
14750        assert_eq!(
14751            ActivityOptions::new()
14752                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
14753                .validate()
14754                .expect_err("protocol integer overflow")
14755                .kind,
14756            ActivityOptionsErrorKind::TimeoutOverflow
14757        );
14758    }
14759
14760    #[test]
14761    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
14762        let ctx = workflow_context(completed_retry_activity_history());
14763        let mut call =
14764            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
14765        let mut task_context = TaskContext::from_waker(noop_waker_ref());
14766
14767        assert!(matches!(
14768            call.as_mut().poll(&mut task_context),
14769            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
14770        ));
14771        assert!(ctx.take_commands().expect("commands").is_empty());
14772        ctx.ensure_history_consumed().expect("history consumed");
14773    }
14774
14775    #[test]
14776    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
14777        let mut options = retry_activity_options();
14778        options
14779            .retry_policy
14780            .as_mut()
14781            .expect("retry policy")
14782            .non_retryable_error_types
14783            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
14784
14785        let new_ctx = workflow_context(Vec::new());
14786        let mut new_call =
14787            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
14788        let mut task_context = TaskContext::from_waker(noop_waker_ref());
14789        assert!(matches!(
14790            new_call.as_mut().poll(&mut task_context),
14791            Poll::Pending
14792        ));
14793        let commands = new_ctx.take_commands().expect("commands");
14794        assert_eq!(commands.len(), 1);
14795        assert_eq!(
14796            commands[0]["retry_policy"]["non_retryable_error_types"],
14797            json!(["PermanentError"])
14798        );
14799
14800        let replay_ctx = workflow_context(completed_retry_activity_history());
14801        let mut replay_call =
14802            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
14803        assert!(matches!(
14804            replay_call.as_mut().poll(&mut task_context),
14805            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
14806        ));
14807        assert!(replay_ctx.take_commands().expect("commands").is_empty());
14808        replay_ctx
14809            .ensure_history_consumed()
14810            .expect("history consumed");
14811    }
14812
14813    #[test]
14814    fn replayed_intermediate_retry_remains_pending_across_restarts() {
14815        let history = completed_retry_activity_history()
14816            .into_iter()
14817            .take(3)
14818            .collect::<Vec<_>>();
14819
14820        for _restart in 0..2 {
14821            let ctx = workflow_context(history.clone());
14822            let mut call =
14823                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
14824            let mut task_context = TaskContext::from_waker(noop_waker_ref());
14825            assert!(matches!(
14826                call.as_mut().poll(&mut task_context),
14827                Poll::Pending
14828            ));
14829            assert!(ctx.take_commands().expect("commands").is_empty());
14830        }
14831    }
14832
14833    #[test]
14834    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
14835        let mut changed_queue = retry_activity_options();
14836        changed_queue.task_queue = Some("different-queue".to_string());
14837
14838        let mut changed_max_attempts = retry_activity_options();
14839        let retry_policy = changed_max_attempts
14840            .retry_policy
14841            .as_mut()
14842            .expect("retry policy");
14843        retry_policy.max_attempts = Some(4);
14844
14845        let mut changed_backoff = retry_activity_options();
14846        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
14847        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
14848            Duration::from_secs(3),
14849            Duration::from_secs(4),
14850        ]));
14851
14852        let mut changed_non_retryable_types = retry_activity_options();
14853        let retry_policy = changed_non_retryable_types
14854            .retry_policy
14855            .as_mut()
14856            .expect("retry policy");
14857        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
14858
14859        let mut changed_start_to_close = retry_activity_options();
14860        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
14861        let mut changed_schedule_to_start = retry_activity_options();
14862        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
14863        let mut changed_schedule_to_close = retry_activity_options();
14864        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
14865        let mut changed_heartbeat = retry_activity_options();
14866        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
14867
14868        let cases = [
14869            (changed_queue, "activity_task_queue_mismatch"),
14870            (changed_max_attempts, "activity_retry_policy_mismatch"),
14871            (changed_backoff, "activity_retry_policy_mismatch"),
14872            (
14873                changed_non_retryable_types,
14874                "activity_retry_policy_mismatch",
14875            ),
14876            (changed_start_to_close, "activity_retry_policy_mismatch"),
14877            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
14878            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
14879            (changed_heartbeat, "activity_retry_policy_mismatch"),
14880        ];
14881
14882        for (options, expected_reason) in cases {
14883            let ctx = workflow_context(completed_retry_activity_history());
14884            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
14885            let mut task_context = TaskContext::from_waker(noop_waker_ref());
14886            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
14887                call.as_mut().poll(&mut task_context)
14888            else {
14889                panic!("changed activity options must fail replay");
14890            };
14891            assert_eq!(failure.reason, expected_reason);
14892            assert_eq!(failure.sequence, Some(1));
14893            assert!(ctx.take_commands().expect("commands").is_empty());
14894        }
14895    }
14896
14897    #[test]
14898    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
14899        let cases = [
14900            (
14901                "execution_mode",
14902                json!("local"),
14903                "activity_execution_mode_mismatch",
14904            ),
14905            (
14906                "snapshot_version",
14907                json!(2),
14908                "activity_retry_policy_mismatch",
14909            ),
14910        ];
14911
14912        for (field, value, expected_reason) in cases {
14913            let mut history = completed_retry_activity_history();
14914            let activity = history[0].payload["activity"]
14915                .as_object_mut()
14916                .expect("activity snapshot");
14917            if field == "execution_mode" {
14918                activity.insert(field.to_string(), value);
14919            } else {
14920                activity["retry_policy"]
14921                    .as_object_mut()
14922                    .expect("retry snapshot")
14923                    .insert(field.to_string(), value);
14924            }
14925
14926            let ctx = workflow_context(history);
14927            let mut call =
14928                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
14929            let mut task_context = TaskContext::from_waker(noop_waker_ref());
14930            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
14931                call.as_mut().poll(&mut task_context)
14932            else {
14933                panic!("changed {field} must fail replay");
14934            };
14935            assert_eq!(failure.reason, expected_reason);
14936            assert_eq!(failure.sequence, Some(1));
14937            assert!(ctx.take_commands().expect("commands").is_empty());
14938        }
14939    }
14940
14941    #[test]
14942    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
14943        let mut history = completed_retry_activity_history();
14944        let activity = history[0].payload["activity"]
14945            .as_object_mut()
14946            .expect("activity snapshot");
14947        activity.remove("execution_mode");
14948        activity.remove("retry_policy");
14949
14950        let mut current = retry_activity_options();
14951        current.start_to_close_timeout = Some(Duration::from_secs(45));
14952        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
14953        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
14954        current.heartbeat_timeout = Some(Duration::from_secs(12));
14955
14956        let ctx = workflow_context(history);
14957        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
14958        let mut task_context = TaskContext::from_waker(noop_waker_ref());
14959        assert!(matches!(
14960            call.as_mut().poll(&mut task_context),
14961            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
14962        ));
14963        assert!(ctx.take_commands().expect("commands").is_empty());
14964        ctx.ensure_history_consumed().expect("history consumed");
14965    }
14966
14967    #[test]
14968    fn terminal_activity_failed_after_start_returns_typed_failure() {
14969        let history = vec![
14970            history_event(
14971                "ActivityScheduled",
14972                json!({
14973                    "sequence": 1,
14974                    "activity_type": "flaky",
14975                    "activity_execution_id": "act-terminal",
14976                    "activity": {
14977                        "id": "act-terminal",
14978                        "sequence": 1,
14979                        "type": "flaky",
14980                        "queue": "critical-activities",
14981                        "retry_policy": {
14982                            "snapshot_version": 1,
14983                            "max_attempts": 3,
14984                            "backoff_seconds": [2, 4],
14985                            "non_retryable_error_types": ["PermanentError"]
14986                        }
14987                    }
14988                }),
14989            ),
14990            history_event(
14991                "ActivityStarted",
14992                json!({
14993                    "sequence": 1,
14994                    "activity_type": "flaky",
14995                    "activity_execution_id": "act-terminal",
14996                    "activity_attempt_id": "attempt-1",
14997                    "attempt_number": 1
14998                }),
14999            ),
15000            history_event(
15001                "ActivityFailed",
15002                json!({
15003                    "sequence": 1,
15004                    "activity_type": "flaky",
15005                    "activity_execution_id": "act-terminal",
15006                    "activity_attempt_id": "attempt-1",
15007                    "attempt_number": 1,
15008                    "failure_id": "failure-terminal",
15009                    "failure_category": "activity",
15010                    "exception_type": "PermanentError",
15011                    "message": "cannot retry",
15012                    "non_retryable": true
15013                }),
15014            ),
15015        ];
15016        let ctx = workflow_context(history);
15017        let mut call =
15018            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
15019        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15020
15021        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15022            call.as_mut().poll(&mut task_context)
15023        else {
15024            panic!("terminal ActivityFailed must settle the activity future");
15025        };
15026        assert_eq!(failure.kind, ActivityFailureKind::Failed);
15027        assert_eq!(
15028            failure.activity_execution_id.as_deref(),
15029            Some("act-terminal")
15030        );
15031        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
15032        assert!(failure.non_retryable);
15033        assert!(ctx.take_commands().expect("commands").is_empty());
15034        ctx.ensure_history_consumed().expect("history consumed");
15035    }
15036
15037    #[test]
15038    fn activity_terminal_events_return_machine_readable_failures() {
15039        let cases = [
15040            (
15041                "ActivityFailed",
15042                json!({
15043                    "sequence": 1,
15044                    "activity_type": "charge-card",
15045                    "activity_execution_id": "act-1",
15046                    "activity_attempt_id": "attempt-2",
15047                    "attempt_number": 2,
15048                    "failure_id": "failure-1",
15049                    "failure_category": "activity",
15050                    "exception_type": "PaymentDeclined",
15051                    "exception_class": "payments.PaymentDeclined",
15052                    "message": "card declined",
15053                    "non_retryable": true
15054                }),
15055                ActivityFailureKind::Failed,
15056                "activity",
15057            ),
15058            (
15059                "ActivityCancelled",
15060                json!({
15061                    "sequence": 1,
15062                    "activity_type": "charge-card",
15063                    "activity_execution_id": "act-1",
15064                    "activity_attempt_id": "attempt-1"
15065                }),
15066                ActivityFailureKind::Cancelled,
15067                "cancelled",
15068            ),
15069        ];
15070
15071        for (event_type, payload, expected_kind, expected_reason) in cases {
15072            let ctx = workflow_context(vec![history_event(event_type, payload)]);
15073            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
15074            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15075            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15076                call.as_mut().poll(&mut task_context)
15077            else {
15078                panic!("expected terminal activity failure");
15079            };
15080            assert_eq!(failure.kind, expected_kind);
15081            assert_eq!(failure.reason, expected_reason);
15082            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
15083            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
15084        }
15085    }
15086
15087    #[test]
15088    fn every_activity_timeout_class_is_typed() {
15089        for timeout_kind in [
15090            "start_to_close",
15091            "schedule_to_start",
15092            "schedule_to_close",
15093            "heartbeat",
15094        ] {
15095            let ctx = workflow_context(vec![history_event(
15096                "ActivityTimedOut",
15097                json!({
15098                    "sequence": 1,
15099                    "activity_type": "slow",
15100                    "activity_execution_id": "act-timeout",
15101                    "activity_attempt_id": "attempt-timeout",
15102                    "failure_category": "timeout",
15103                    "timeout_kind": timeout_kind,
15104                    "message": "deadline expired"
15105                }),
15106            )]);
15107            let mut call = Box::pin(ctx.activity("slow", json!([])));
15108            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15109            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
15110                call.as_mut().poll(&mut task_context)
15111            else {
15112                panic!("expected timeout failure");
15113            };
15114            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
15115            assert_eq!(failure.reason, timeout_kind);
15116            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
15117            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
15118        }
15119    }
15120
15121    #[test]
15122    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
15123        let ctx = workflow_context(Vec::new());
15124        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
15125        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15126
15127        assert!(matches!(
15128            sleep.as_mut().poll(&mut task_context),
15129            Poll::Pending
15130        ));
15131        assert!(matches!(
15132            sleep.as_mut().poll(&mut task_context),
15133            Poll::Pending
15134        ));
15135
15136        let commands = ctx.take_commands().expect("timer command");
15137        assert_eq!(
15138            commands,
15139            vec![json!({
15140                "type": "start_timer",
15141                "delay_seconds": 2,
15142            })]
15143        );
15144    }
15145
15146    #[test]
15147    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
15148        let history = vec![
15149            history_event(
15150                "TimerScheduled",
15151                json!({
15152                    "sequence": 1,
15153                    "timer_id": "timer-1",
15154                    "delay_seconds": 5,
15155                    "fire_at": "2026-07-11T12:00:05Z",
15156                }),
15157            ),
15158            history_event(
15159                "TimerFired",
15160                json!({
15161                    "sequence": 1,
15162                    "timer_id": "timer-1",
15163                    "delay_seconds": 5,
15164                    "fire_at": "2026-07-11T12:00:05Z",
15165                    "fired_at": "2026-07-11T12:00:05Z",
15166                }),
15167            ),
15168        ];
15169
15170        for _restart in 0..2 {
15171            let ctx = workflow_context(history.clone());
15172            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
15173            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15174            assert!(matches!(
15175                sleep.as_mut().poll(&mut task_context),
15176                Poll::Ready(Ok(()))
15177            ));
15178            assert!(ctx.take_commands().expect("commands").is_empty());
15179            ctx.ensure_history_consumed().expect("history consumed");
15180        }
15181    }
15182
15183    #[test]
15184    fn workflow_sleep_rejects_changed_delay_during_replay() {
15185        let ctx = workflow_context(vec![
15186            history_event(
15187                "TimerScheduled",
15188                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15189            ),
15190            history_event(
15191                "TimerFired",
15192                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15193            ),
15194        ]);
15195        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
15196        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15197
15198        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15199            sleep.as_mut().poll(&mut task_context)
15200        else {
15201            panic!("changed timer delay must be rejected");
15202        };
15203        assert_eq!(failure.reason, "timer_delay_mismatch");
15204        assert_eq!(failure.sequence, Some(1));
15205    }
15206
15207    #[test]
15208    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
15209        let ctx = workflow_context(Vec::new());
15210        let mut wait = Box::pin(
15211            ctx.wait_condition(
15212                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
15213                    .timeout(Duration::from_millis(60_001)),
15214                || Ok(false),
15215            ),
15216        );
15217        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15218
15219        assert!(matches!(
15220            wait.as_mut().poll(&mut task_context),
15221            Poll::Pending
15222        ));
15223        assert!(matches!(
15224            wait.as_mut().poll(&mut task_context),
15225            Poll::Pending
15226        ));
15227        assert_eq!(
15228            ctx.take_commands().expect("condition command"),
15229            vec![json!({
15230                "type": "open_condition_wait",
15231                "condition_key": "approval.ready",
15232                "condition_definition_fingerprint": "sha256:approval-v1",
15233                "timeout_seconds": 61,
15234            })]
15235        );
15236    }
15237
15238    #[test]
15239    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
15240        let ctx = workflow_context(Vec::new());
15241        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
15242        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15243        assert!(matches!(
15244            satisfied.as_mut().poll(&mut task_context),
15245            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15246        ));
15247
15248        let mut timed_out = Box::pin(wait_condition!(
15249            ctx,
15250            "no-wait",
15251            timeout: Duration::ZERO,
15252            || Ok(false),
15253        ));
15254        assert!(matches!(
15255            timed_out.as_mut().poll(&mut task_context),
15256            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
15257        ));
15258        assert!(ctx.take_commands().expect("commands").is_empty());
15259    }
15260
15261    #[test]
15262    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
15263        let signal_history = vec![
15264            history_event(
15265                "ConditionWaitOpened",
15266                json!({
15267                    "sequence": 4,
15268                    "condition_wait_id": "condition:4",
15269                    "condition_key": "approval",
15270                    "condition_definition_fingerprint": "sha256:approval-v1",
15271                    "timeout_seconds": 30,
15272                }),
15273            ),
15274            history_event(
15275                "SignalReceived",
15276                json!({
15277                    "workflow_sequence": 4,
15278                    "signal_name": "approve",
15279                    "arguments": fixture_envelope(json!(["Ada"])),
15280                }),
15281            ),
15282        ];
15283        for _worker_before_or_after_restart in 0..2 {
15284            let ctx = workflow_context(signal_history.clone());
15285            let predicate_ctx = ctx.clone();
15286            let mut wait = Box::pin(
15287                ctx.wait_condition(
15288                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
15289                        .timeout(Duration::from_secs(30)),
15290                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
15291                ),
15292            );
15293            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15294            assert!(matches!(
15295                wait.as_mut().poll(&mut task_context),
15296                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15297            ));
15298            assert!(ctx.take_commands().expect("commands").is_empty());
15299            ctx.ensure_history_consumed().expect("condition consumed");
15300        }
15301
15302        let update_history = vec![
15303            history_event(
15304                "ConditionWaitOpened",
15305                json!({
15306                    "sequence": 7,
15307                    "condition_wait_id": "condition:7",
15308                    "condition_key": "update-approval",
15309                    "condition_definition_fingerprint": "sha256:update-approval-v1",
15310                }),
15311            ),
15312            history_event(
15313                "UpdateApplied",
15314                json!({
15315                    "sequence": 7,
15316                    "update_id": "update-1",
15317                    "update_name": "approve",
15318                    "arguments": fixture_envelope(json!([true])),
15319                }),
15320            ),
15321        ];
15322        let ctx = workflow_context(update_history);
15323        let predicate_ctx = ctx.clone();
15324        let mut wait = Box::pin(ctx.wait_condition(
15325            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
15326            move || {
15327                Ok(predicate_ctx
15328                    .updates("approve")?
15329                    .first()
15330                    .and_then(|arguments| arguments.first())
15331                    .and_then(Value::as_bool)
15332                    == Some(true))
15333            },
15334        ));
15335        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15336        assert!(matches!(
15337            wait.as_mut().poll(&mut task_context),
15338            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15339        ));
15340        assert!(ctx.take_commands().expect("commands").is_empty());
15341        ctx.ensure_history_consumed().expect("condition consumed");
15342    }
15343
15344    #[test]
15345    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
15346        let open_history = vec![
15347            history_event(
15348                "ConditionWaitOpened",
15349                json!({
15350                    "sequence": 3,
15351                    "condition_wait_id": "condition:3",
15352                    "condition_key": "two-votes",
15353                    "condition_definition_fingerprint": "sha256:two-votes-v1",
15354                    "timeout_seconds": 120,
15355                }),
15356            ),
15357            history_event(
15358                "SignalReceived",
15359                json!({
15360                    "workflow_sequence": 3,
15361                    "signal_name": "vote",
15362                    "arguments": fixture_envelope(json!(["first"])),
15363                }),
15364            ),
15365        ];
15366        for _worker_before_or_after_restart in 0..2 {
15367            let ctx = workflow_context(open_history.clone());
15368            let predicate_ctx = ctx.clone();
15369            let mut wait = Box::pin(
15370                ctx.wait_condition(
15371                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
15372                        .timeout(Duration::from_secs(120)),
15373                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
15374                ),
15375            );
15376            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15377            assert!(matches!(
15378                wait.as_mut().poll(&mut task_context),
15379                Poll::Pending
15380            ));
15381            assert_eq!(
15382                ctx.take_commands().expect("reopened condition"),
15383                vec![json!({
15384                    "type": "open_condition_wait",
15385                    "condition_key": "two-votes",
15386                    "condition_definition_fingerprint": "sha256:two-votes-v1",
15387                    "timeout_seconds": 120,
15388                })]
15389            );
15390        }
15391
15392        let satisfied_ctx = workflow_context(vec![
15393            history_event(
15394                "ConditionWaitOpened",
15395                json!({
15396                    "sequence": 5,
15397                    "condition_wait_id": "condition:5",
15398                    "condition_key": "approval",
15399                    "condition_definition_fingerprint": "sha256:approval-v1",
15400                }),
15401            ),
15402            history_event(
15403                "ConditionWaitSatisfied",
15404                json!({
15405                    "sequence": 5,
15406                    "condition_wait_id": "condition:5",
15407                    "condition_key": "approval",
15408                    "condition_definition_fingerprint": "sha256:approval-v1",
15409                }),
15410            ),
15411        ]);
15412        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
15413            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
15414            || Ok(false),
15415        ));
15416        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15417        assert!(matches!(
15418            satisfied.as_mut().poll(&mut task_context),
15419            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15420        ));
15421
15422        let timed_out_ctx = workflow_context(vec![
15423            history_event(
15424                "ConditionWaitOpened",
15425                json!({
15426                    "sequence": 8,
15427                    "condition_wait_id": "condition:8",
15428                    "condition_key": "approval-timeout",
15429                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
15430                    "timeout_seconds": 5,
15431                }),
15432            ),
15433            history_event(
15434                "TimerScheduled",
15435                json!({
15436                    "sequence": 9,
15437                    "timer_id": "condition-timer:9",
15438                    "timer_kind": "condition_timeout",
15439                    "condition_wait_id": "condition:8",
15440                    "delay_seconds": 5,
15441                }),
15442            ),
15443            history_event(
15444                "TimerFired",
15445                json!({
15446                    "sequence": 9,
15447                    "timer_id": "condition-timer:9",
15448                    "timer_kind": "condition_timeout",
15449                    "condition_wait_id": "condition:8",
15450                    "delay_seconds": 5,
15451                }),
15452            ),
15453        ]);
15454        let mut timed_out = Box::pin(
15455            timed_out_ctx.wait_condition(
15456                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
15457                    .timeout(Duration::from_secs(5)),
15458                || Ok(true),
15459            ),
15460        );
15461        assert!(matches!(
15462            timed_out.as_mut().poll(&mut task_context),
15463            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
15464        ));
15465    }
15466
15467    #[test]
15468    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
15469        let ctx = workflow_context(vec![
15470            history_event(
15471                "ConditionWaitOpened",
15472                json!({
15473                    "sequence": 3,
15474                    "condition_wait_id": "condition:3",
15475                    "condition_key": "two-votes",
15476                    "condition_definition_fingerprint": "sha256:two-votes-v1",
15477                }),
15478            ),
15479            history_event(
15480                "SignalReceived",
15481                json!({
15482                    "workflow_sequence": 3,
15483                    "signal_name": "vote",
15484                    "arguments": fixture_envelope(json!(["first"])),
15485                }),
15486            ),
15487            history_event(
15488                "ConditionWaitSatisfied",
15489                json!({
15490                    "sequence": 3,
15491                    "condition_wait_id": "condition:3",
15492                    "condition_key": "two-votes",
15493                    "condition_definition_fingerprint": "sha256:two-votes-v1",
15494                }),
15495            ),
15496            history_event(
15497                "ConditionWaitOpened",
15498                json!({
15499                    "sequence": 5,
15500                    "condition_wait_id": "condition:5",
15501                    "condition_key": "two-votes",
15502                    "condition_definition_fingerprint": "sha256:two-votes-v1",
15503                }),
15504            ),
15505            history_event(
15506                "SignalReceived",
15507                json!({
15508                    "workflow_sequence": 5,
15509                    "signal_name": "vote",
15510                    "arguments": fixture_envelope(json!(["second"])),
15511                }),
15512            ),
15513            history_event(
15514                "ConditionWaitSatisfied",
15515                json!({
15516                    "sequence": 5,
15517                    "condition_wait_id": "condition:5",
15518                    "condition_key": "two-votes",
15519                    "condition_definition_fingerprint": "sha256:two-votes-v1",
15520                }),
15521            ),
15522        ]);
15523        let predicate_ctx = ctx.clone();
15524        let mut wait = Box::pin(ctx.wait_condition(
15525            ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
15526            move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
15527        ));
15528        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15529
15530        assert!(matches!(
15531            wait.as_mut().poll(&mut task_context),
15532            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
15533        ));
15534        ctx.ensure_history_consumed()
15535            .expect("every physical wait-open is consumed");
15536    }
15537
15538    #[test]
15539    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
15540        let history = vec![history_event(
15541            "ConditionWaitOpened",
15542            json!({
15543                "sequence": 12,
15544                "condition_wait_id": "condition:12",
15545                "condition_key": "approval",
15546                "condition_definition_fingerprint": "sha256:approval-v1",
15547                "timeout_seconds": 30,
15548            }),
15549        )];
15550        for (options, expected_reason) in [
15551            (
15552                ConditionWaitOptions::new("changed", "sha256:approval-v1")
15553                    .timeout(Duration::from_secs(30)),
15554                "condition_wait_key_mismatch",
15555            ),
15556            (
15557                ConditionWaitOptions::new("approval", "sha256:approval-v2")
15558                    .timeout(Duration::from_secs(30)),
15559                "condition_wait_predicate_mismatch",
15560            ),
15561            (
15562                ConditionWaitOptions::new("approval", "sha256:approval-v1")
15563                    .timeout(Duration::from_secs(29)),
15564                "condition_wait_timeout_mismatch",
15565            ),
15566        ] {
15567            let ctx = workflow_context(history.clone());
15568            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
15569            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15570            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15571                wait.as_mut().poll(&mut task_context)
15572            else {
15573                panic!("changed condition definition must fail replay");
15574            };
15575            assert_eq!(failure.reason, expected_reason);
15576            assert_eq!(failure.sequence, Some(12));
15577        }
15578    }
15579
15580    #[test]
15581    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
15582        let error = WorkflowState::new(
15583            vec![history_event(
15584                "ConditionWaitOpened",
15585                json!({
15586                    "sequence": 12,
15587                    "condition_wait_id": "condition:12",
15588                    "condition_key": "approval",
15589                }),
15590            )],
15591            "rust-workers".to_string(),
15592            DEFAULT_CODEC.to_string(),
15593            None,
15594        )
15595        .expect_err("condition history without a predicate fingerprint must fail");
15596
15597        assert!(matches!(
15598            error,
15599            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15600                if reason == "condition_wait_predicate_fingerprint_missing"
15601        ));
15602    }
15603
15604    #[test]
15605    fn typed_search_attribute_updates_validate_emit_and_replay() {
15606        let update = SearchAttributeUpdate::new()
15607            .keyword("OrderStatus", " waiting ")
15608            .expect("keyword")
15609            .int("Attempt", 3)
15610            .expect("int")
15611            .bool("Escalated", false)
15612            .expect("bool")
15613            .keyword_list("Regions", ["us-east", "eu-west"])
15614            .expect("list")
15615            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
15616            .expect("datetime")
15617            .delete("LegacyStatus")
15618            .expect("delete");
15619        let ctx = workflow_context(Vec::new());
15620        ctx.upsert_search_attributes(update.clone())
15621            .expect("typed update");
15622        assert_eq!(
15623            ctx.take_commands().expect("search-attribute command"),
15624            vec![json!({
15625                "type": "upsert_search_attributes",
15626                "attributes": {
15627                    "Attempt": 3,
15628                    "Escalated": false,
15629                    "LegacyStatus": null,
15630                    "OrderStatus": "waiting",
15631                    "Regions": ["us-east", "eu-west"],
15632                    "UpdatedAt": "2026-08-22T04:00:00Z",
15633                },
15634                "attribute_types": {
15635                    "Attempt": "int",
15636                    "Escalated": "bool",
15637                    "OrderStatus": "keyword",
15638                    "Regions": "keyword_list",
15639                    "UpdatedAt": "datetime",
15640                },
15641            })]
15642        );
15643
15644        let replay = workflow_context(vec![history_event(
15645            "SearchAttributesUpserted",
15646            json!({
15647                "sequence": 6,
15648                "attributes": {
15649                    "Attempt": 3,
15650                    "Escalated": false,
15651                    "LegacyStatus": null,
15652                    "OrderStatus": "waiting",
15653                    "Regions": ["us-east", "eu-west"],
15654                    "UpdatedAt": "2026-08-22T04:00:00Z",
15655                },
15656                "attribute_types": {
15657                    "Attempt": "int",
15658                    "Escalated": "bool",
15659                    "OrderStatus": "keyword",
15660                    "Regions": "keyword_list",
15661                    "UpdatedAt": "datetime",
15662                },
15663                "merged": {},
15664            }),
15665        )]);
15666        replay
15667            .upsert_search_attributes(update)
15668            .expect("matching update replays");
15669        assert!(replay.take_commands().expect("commands").is_empty());
15670        replay.ensure_history_consumed().expect("history consumed");
15671
15672        let type_drift = workflow_context(vec![history_event(
15673            "SearchAttributesUpserted",
15674            json!({
15675                "sequence": 7,
15676                "attributes": {"OrderStatus": "waiting"},
15677                "attribute_types": {"OrderStatus": "keyword"},
15678                "merged": {"OrderStatus": "waiting"},
15679            }),
15680        )]);
15681        let error = type_drift
15682            .upsert_search_attributes(
15683                SearchAttributeUpdate::new()
15684                    .string("OrderStatus", "waiting")
15685                    .expect("string update"),
15686            )
15687            .expect_err("same JSON value with a changed type must fail replay");
15688        assert!(matches!(
15689            error,
15690            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15691                if reason == "search_attribute_type_mismatch"
15692        ));
15693
15694        let malformed_types = WorkflowState::new(
15695            vec![history_event(
15696                "SearchAttributesUpserted",
15697                json!({
15698                    "sequence": 8,
15699                    "attributes": {"OrderStatus": "waiting"},
15700                    "attribute_types": {"OrderStatus": "unsupported"},
15701                    "merged": {"OrderStatus": "waiting"},
15702                }),
15703            )],
15704            "rust-workers".to_string(),
15705            DEFAULT_CODEC.to_string(),
15706            None,
15707        )
15708        .expect_err("unsupported search-attribute type metadata must fail");
15709        assert!(matches!(
15710            malformed_types,
15711            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15712                if reason == "search_attribute_types_malformed"
15713        ));
15714
15715        assert!(matches!(
15716            SearchAttributeUpdate::new().keyword("bad key", "value"),
15717            Err(SearchAttributeUpdateError::InvalidKey(_))
15718        ));
15719        assert!(matches!(
15720            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
15721            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
15722        ));
15723        assert!(matches!(
15724            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
15725            Err(SearchAttributeUpdateError::ValueTooLong { .. })
15726        ));
15727        assert!(matches!(
15728            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
15729            Err(SearchAttributeUpdateError::InvalidDateTime(_))
15730        ));
15731        assert!(matches!(
15732            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
15733            Err(Error::InvalidSearchAttributeUpdate(
15734                SearchAttributeUpdateError::Empty
15735            ))
15736        ));
15737    }
15738
15739    #[test]
15740    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
15741        let lone_fire = WorkflowState::new(
15742            vec![history_event(
15743                "TimerFired",
15744                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15745            )],
15746            "rust-workers".to_string(),
15747            DEFAULT_CODEC.to_string(),
15748            None,
15749        )
15750        .expect_err("TimerFired requires TimerScheduled");
15751        assert!(matches!(
15752            lone_fire,
15753            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15754                if reason == "timer_schedule_missing_or_duplicate"
15755        ));
15756
15757        let wrong_identity = WorkflowState::new(
15758            vec![
15759                history_event(
15760                    "TimerScheduled",
15761                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15762                ),
15763                history_event(
15764                    "TimerFired",
15765                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
15766                ),
15767            ],
15768            "rust-workers".to_string(),
15769            DEFAULT_CODEC.to_string(),
15770            None,
15771        )
15772        .expect_err("fire must match scheduled timer identity");
15773        assert!(matches!(
15774            wrong_identity,
15775            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15776                if reason == "timer_identity_mismatch"
15777        ));
15778
15779        let duplicate_fire = WorkflowState::new(
15780            vec![
15781                history_event(
15782                    "TimerScheduled",
15783                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15784                ),
15785                history_event(
15786                    "TimerFired",
15787                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15788                ),
15789                history_event(
15790                    "TimerFired",
15791                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15792                ),
15793            ],
15794            "rust-workers".to_string(),
15795            DEFAULT_CODEC.to_string(),
15796            None,
15797        )
15798        .expect_err("a durable timer cannot fire twice");
15799        assert!(matches!(
15800            duplicate_fire,
15801            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15802                if reason == "duplicate_timer_fire"
15803        ));
15804
15805        let wrong_fired_delay = WorkflowState::new(
15806            vec![
15807                history_event(
15808                    "TimerScheduled",
15809                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15810                ),
15811                history_event(
15812                    "TimerFired",
15813                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
15814                ),
15815            ],
15816            "rust-workers".to_string(),
15817            DEFAULT_CODEC.to_string(),
15818            None,
15819        )
15820        .expect_err("timer schedule and fire delays must agree");
15821        assert!(matches!(
15822            wrong_fired_delay,
15823            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
15824                if reason == "timer_history_delay_mismatch"
15825        ));
15826    }
15827
15828    #[test]
15829    fn replay_rejects_activity_moved_before_recorded_timer() {
15830        let ctx = workflow_context(vec![
15831            history_event(
15832                "TimerScheduled",
15833                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15834            ),
15835            history_event(
15836                "TimerFired",
15837                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15838            ),
15839            history_event(
15840                "ActivityCompleted",
15841                json!({
15842                    "sequence": 2,
15843                    "activity_type": "after-timer",
15844                    "payload_codec": DEFAULT_CODEC,
15845                    "result": fixture_envelope(json!("done")),
15846                }),
15847            ),
15848        ]);
15849        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
15850        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15851
15852        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15853            activity.as_mut().poll(&mut task_context)
15854        else {
15855            panic!("reordered durable command must be rejected");
15856        };
15857        assert_eq!(failure.reason, "recorded_command_mismatch");
15858        assert_eq!(failure.sequence, Some(1));
15859        assert_eq!(failure.expected.as_deref(), Some("timer"));
15860        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
15861    }
15862
15863    #[test]
15864    fn workflow_context_emits_a_typed_named_signal_wait() {
15865        let ctx = workflow_context(Vec::new());
15866        let mut signal = Box::pin(ctx.wait_signal("finish"));
15867        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15868
15869        assert!(matches!(
15870            signal.as_mut().poll(&mut task_context),
15871            Poll::Pending
15872        ));
15873        assert_eq!(
15874            ctx.take_commands().expect("signal-wait command"),
15875            vec![json!({
15876                "type": "open_signal_wait",
15877                "signal_name": "finish",
15878            })]
15879        );
15880    }
15881
15882    #[test]
15883    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
15884        let ctx = workflow_context(vec![
15885            history_event(
15886                "ConditionWaitOpened",
15887                json!({
15888                    "sequence": 1,
15889                    "condition_wait_id": "condition:1",
15890                    "condition_key": "signal:finish",
15891                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
15892                }),
15893            ),
15894            history_event(
15895                "ConditionWaitSatisfied",
15896                json!({
15897                    "sequence": 1,
15898                    "condition_wait_id": "condition:1",
15899                    "condition_key": "signal:finish",
15900                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
15901                }),
15902            ),
15903            history_event(
15904                "SignalReceived",
15905                json!({"signal_name": "finish", "arguments": []}),
15906            ),
15907        ]);
15908        let mut signal = Box::pin(ctx.wait_signal("finish"));
15909        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15910
15911        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15912            signal.as_mut().poll(&mut task_context)
15913        else {
15914            panic!("condition history must not resolve as a typed signal wait");
15915        };
15916        assert_eq!(failure.reason, "recorded_command_mismatch");
15917        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
15918    }
15919
15920    #[test]
15921    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
15922        let signal_then_timer = vec![
15923            history_event(
15924                "SignalWaitOpened",
15925                json!({"sequence": 1, "signal_name": "go"}),
15926            ),
15927            history_event(
15928                "SignalApplied",
15929                json!({
15930                    "sequence": 1,
15931                    "signal_name": "go",
15932                    "value": fixture_envelope(json!(["now"])),
15933                }),
15934            ),
15935            history_event(
15936                "TimerScheduled",
15937                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
15938            ),
15939            history_event(
15940                "TimerFired",
15941                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
15942            ),
15943        ];
15944
15945        let ctx = workflow_context(signal_then_timer.clone());
15946        let mut signal = Box::pin(ctx.wait_signal("go"));
15947        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15948        assert!(matches!(
15949            signal.as_mut().poll(&mut task_context),
15950            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
15951        ));
15952        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
15953        assert!(matches!(
15954            timer.as_mut().poll(&mut task_context),
15955            Poll::Ready(Ok(()))
15956        ));
15957        ctx.ensure_history_consumed()
15958            .expect("signal and timer history consumed in order");
15959
15960        let reordered = workflow_context(signal_then_timer);
15961        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
15962        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15963            timer_first.as_mut().poll(&mut task_context)
15964        else {
15965            panic!("timer cannot consume signal-wait-first history");
15966        };
15967        assert_eq!(failure.reason, "recorded_command_mismatch");
15968        assert_eq!(failure.sequence, Some(1));
15969        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
15970
15971        let timer_then_signal = vec![
15972            history_event(
15973                "TimerScheduled",
15974                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15975            ),
15976            history_event(
15977                "TimerFired",
15978                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
15979            ),
15980            history_event(
15981                "SignalWaitOpened",
15982                json!({"sequence": 2, "signal_name": "go"}),
15983            ),
15984            history_event(
15985                "SignalApplied",
15986                json!({
15987                    "sequence": 2,
15988                    "signal_name": "go",
15989                    "value": fixture_envelope(json!([])),
15990                }),
15991            ),
15992        ];
15993        let reordered = workflow_context(timer_then_signal);
15994        let mut signal_first = Box::pin(reordered.wait_signal("go"));
15995        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
15996            signal_first.as_mut().poll(&mut task_context)
15997        else {
15998            panic!("signal wait cannot consume timer-first history");
15999        };
16000        assert_eq!(failure.reason, "recorded_command_mismatch");
16001        assert_eq!(failure.sequence, Some(1));
16002        assert_eq!(failure.expected.as_deref(), Some("timer"));
16003    }
16004
16005    #[test]
16006    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
16007        let duplicate_timer = WorkflowState::new(
16008            vec![
16009                history_event(
16010                    "TimerScheduled",
16011                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16012                ),
16013                history_event(
16014                    "TimerScheduled",
16015                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
16016                ),
16017            ],
16018            "rust-workers".to_string(),
16019            DEFAULT_CODEC.to_string(),
16020            None,
16021        )
16022        .expect_err("one workflow sequence cannot schedule two timers");
16023        assert!(matches!(
16024            duplicate_timer,
16025            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16026                if reason == "timer_schedule_missing_or_duplicate"
16027        ));
16028
16029        let colliding_kinds = WorkflowState::new(
16030            vec![
16031                history_event(
16032                    "TimerScheduled",
16033                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16034                ),
16035                history_event(
16036                    "ActivityCompleted",
16037                    json!({"sequence": 1, "activity_type": "same-sequence"}),
16038                ),
16039            ],
16040            "rust-workers".to_string(),
16041            DEFAULT_CODEC.to_string(),
16042            None,
16043        )
16044        .expect_err("one workflow sequence cannot identify two command kinds");
16045        assert!(matches!(
16046            colliding_kinds,
16047            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16048                if reason == "durable_command_sequence_collision"
16049        ));
16050
16051        let duplicate_signal_wait = WorkflowState::new(
16052            vec![
16053                history_event(
16054                    "SignalWaitOpened",
16055                    json!({"sequence": 1, "signal_name": "go"}),
16056                ),
16057                history_event(
16058                    "SignalWaitOpened",
16059                    json!({"sequence": 1, "signal_name": "go"}),
16060                ),
16061            ],
16062            "rust-workers".to_string(),
16063            DEFAULT_CODEC.to_string(),
16064            None,
16065        )
16066        .expect_err("one workflow sequence cannot open two signal waits");
16067        assert!(matches!(
16068            duplicate_signal_wait,
16069            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16070                if reason == "signal_wait_open_missing_or_duplicate"
16071        ));
16072    }
16073
16074    #[test]
16075    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
16076        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
16077            .expect("side-effect result");
16078        let ctx = workflow_context(vec![history_event(
16079            "SideEffectRecorded",
16080            json!({"sequence": 99, "result": result}),
16081        )]);
16082
16083        let replayed: Value = ctx
16084            .side_effect(|| panic!("recorded side effect must not run"))
16085            .expect("positive global workflow sequence is valid");
16086        assert_eq!(replayed, json!({"captured": true}));
16087        ctx.ensure_history_consumed().expect("history consumed");
16088    }
16089
16090    #[test]
16091    fn workflow_history_rejects_zero_and_descending_command_sequences() {
16092        let result =
16093            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
16094        let zero = WorkflowState::new(
16095            vec![history_event(
16096                "SideEffectRecorded",
16097                json!({"sequence": 0, "result": result.clone()}),
16098            )],
16099            "rust-workers".to_string(),
16100            DEFAULT_CODEC.to_string(),
16101            None,
16102        )
16103        .expect_err("durable command sequences must be positive");
16104        assert!(matches!(
16105            zero,
16106            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16107                if reason == "durable_command_sequence_invalid"
16108        ));
16109
16110        let descending = WorkflowState::new(
16111            vec![
16112                history_event(
16113                    "SideEffectRecorded",
16114                    json!({"sequence": 3, "result": result}),
16115                ),
16116                history_event(
16117                    "VersionMarkerRecorded",
16118                    json!({
16119                        "sequence": 2,
16120                        "change_id": "descending-marker",
16121                        "version": 1,
16122                        "min_supported": 1,
16123                        "max_supported": 1,
16124                    }),
16125                ),
16126            ],
16127            "rust-workers".to_string(),
16128            DEFAULT_CODEC.to_string(),
16129            None,
16130        )
16131        .expect_err("new durable commands must remain strictly ordered");
16132        let Error::NonDeterministicReplay(failure) = descending else {
16133            panic!("expected typed replay failure");
16134        };
16135        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
16136        assert_eq!(failure.sequence, Some(2));
16137        assert_eq!(
16138            failure.expected.as_deref(),
16139            Some("workflow sequence greater than 3")
16140        );
16141        assert_eq!(failure.actual.as_deref(), Some("2"));
16142    }
16143
16144    #[test]
16145    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
16146        fn worker() -> Worker {
16147            let client = Client::new("http://127.0.0.1:8080").expect("client");
16148            let mut worker = Worker::new(client, "rust-workers");
16149            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
16150                ctx.wait_signal("finish").await?;
16151                let marker: String =
16152                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
16153                assert_eq!(marker, "after-finish");
16154                Ok(json!("finished"))
16155            });
16156            worker
16157        }
16158
16159        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
16160            .expect("side-effect result");
16161        let task = workflow_task(
16162            "rust.finish-after-gaps",
16163            vec![
16164                history_event(
16165                    "SignalWaitOpened",
16166                    json!({"sequence": 1, "signal_name": "finish"}),
16167                ),
16168                history_event(
16169                    "SignalReceived",
16170                    json!({
16171                        "signal_id": "increment-3",
16172                        "signal_name": "increment",
16173                        "workflow_sequence": 2,
16174                        "payload_codec": DEFAULT_CODEC,
16175                        "arguments": fixture_envelope(json!([3])),
16176                    }),
16177                ),
16178                history_event(
16179                    "SignalReceived",
16180                    json!({
16181                        "signal_id": "increment-5",
16182                        "signal_name": "increment",
16183                        "workflow_sequence": 3,
16184                        "payload_codec": DEFAULT_CODEC,
16185                        "arguments": fixture_envelope(json!([5])),
16186                    }),
16187                ),
16188                history_event(
16189                    "SignalReceived",
16190                    json!({
16191                        "signal_id": "finish",
16192                        "signal_name": "finish",
16193                        "workflow_sequence": 4,
16194                        "payload_codec": DEFAULT_CODEC,
16195                        "arguments": fixture_envelope(json!([])),
16196                    }),
16197                ),
16198                history_event(
16199                    "SignalApplied",
16200                    json!({
16201                        "sequence": 1,
16202                        "signal_id": "finish",
16203                        "signal_name": "finish",
16204                        "payload_codec": DEFAULT_CODEC,
16205                        "value": fixture_envelope(json!([])),
16206                    }),
16207                ),
16208                history_event(
16209                    "SideEffectRecorded",
16210                    json!({"sequence": 5, "result": marker}),
16211                ),
16212            ],
16213            DEFAULT_CODEC,
16214        );
16215
16216        for _original_or_cold_worker in 0..2 {
16217            let commands = worker()
16218                .execute_workflow_task(task.clone())
16219                .expect("signal gaps preserve deterministic replay");
16220            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
16221            assert_eq!(commands[0]["type"], "complete_workflow");
16222            assert_eq!(
16223                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
16224                json!("finished")
16225            );
16226        }
16227    }
16228
16229    #[test]
16230    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
16231        let ctx = workflow_context(Vec::new());
16232        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
16233        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16234        assert!(matches!(
16235            sleep.as_mut().poll(&mut task_context),
16236            Poll::Ready(Err(Error::TimerDurationOverflow))
16237        ));
16238        assert!(ctx.take_commands().expect("commands").is_empty());
16239    }
16240
16241    #[test]
16242    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
16243        let entries = AvroValue::Map(BTreeMap::from([
16244            ("text".to_string(), AvroValue::String("same".to_string())),
16245            (
16246                "nested".to_string(),
16247                AvroValue::Map(BTreeMap::from([
16248                    ("beta".to_string(), AvroValue::Long(2)),
16249                    ("alpha".to_string(), AvroValue::Long(1)),
16250                ])),
16251            ),
16252            ("long".to_string(), AvroValue::Long(7)),
16253            ("double".to_string(), AvroValue::Double(7.0)),
16254            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
16255        ]));
16256        let ctx = workflow_context(Vec::new());
16257        ctx.upsert_memo(entries.clone()).expect("valid memo update");
16258        let commands = ctx.take_commands().expect("commands");
16259
16260        assert_eq!(commands.len(), 1);
16261        assert_eq!(commands[0]["type"], "upsert_memo");
16262        let server_entries = json!({
16263            "codec": "avro",
16264            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
16265        });
16266        assert_eq!(
16267            commands[0]["entries"]
16268                .as_object()
16269                .expect("entries envelope")
16270                .keys()
16271                .collect::<Vec<_>>(),
16272            vec!["blob", "codec"]
16273        );
16274        assert_eq!(commands[0]["entries"], server_entries);
16275        let wire_entries =
16276            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
16277        assert_eq!(wire_entries, entries);
16278
16279        let history = vec![history_event(
16280            "MemoUpserted",
16281            json!({
16282                "sequence": 1,
16283                "entries": server_entries.clone(),
16284                "merged": server_entries,
16285            }),
16286        )];
16287        let replay = workflow_context(history.clone());
16288        replay
16289            .upsert_memo(entries.clone())
16290            .expect("matching replay identity");
16291        assert!(replay.take_commands().expect("replay commands").is_empty());
16292
16293        let changed_types = AvroValue::Map(BTreeMap::from([
16294            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
16295            (
16296                "nested".to_string(),
16297                AvroValue::Map(BTreeMap::from([
16298                    ("alpha".to_string(), AvroValue::Long(1)),
16299                    ("beta".to_string(), AvroValue::Long(2)),
16300                ])),
16301            ),
16302            ("long".to_string(), AvroValue::Double(7.0)),
16303            ("double".to_string(), AvroValue::Long(7)),
16304            ("binary".to_string(), AvroValue::String("same".to_string())),
16305        ]));
16306        let error = workflow_context(history)
16307            .upsert_memo(changed_types)
16308            .expect_err("memo replay identity must preserve Avro value types");
16309        assert!(matches!(
16310            error,
16311            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
16312        ));
16313    }
16314
16315    #[test]
16316    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
16317        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
16318            .expect("memo envelope");
16319        let replay = workflow_context(vec![history_event(
16320            "MemoUpserted",
16321            json!({
16322                "sequence": 1,
16323                "entries": original.clone(),
16324                "merged": original
16325            }),
16326        )]);
16327        let error = replay
16328            .upsert_memo(json!({"stage": "changed"}))
16329            .expect_err("changed memo update must fail replay");
16330        assert!(matches!(
16331            error,
16332            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
16333        ));
16334
16335        let invalid = workflow_context(Vec::new())
16336            .upsert_memo(
16337                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
16338            )
16339            .expect_err("oversized key");
16340        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
16341    }
16342
16343    #[test]
16344    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
16345        let negative_zero = AvroValue::Map(BTreeMap::from([(
16346            "reading".to_string(),
16347            AvroValue::Double(-0.0),
16348        )]));
16349        let negative_zero_envelope =
16350            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
16351        let history = vec![history_event(
16352            "MemoUpserted",
16353            json!({
16354                "sequence": 1,
16355                "entries": negative_zero_envelope.clone(),
16356                "merged": negative_zero_envelope,
16357            }),
16358        )];
16359
16360        workflow_context(history.clone())
16361            .upsert_memo(negative_zero)
16362            .expect("matching negative-zero history identity");
16363
16364        let error = workflow_context(history)
16365            .upsert_memo(AvroValue::Map(BTreeMap::from([(
16366                "reading".to_string(),
16367                AvroValue::Double(0.0),
16368            )])))
16369            .expect_err("positive zero must not consume negative-zero memo history");
16370        assert!(matches!(
16371            error,
16372            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
16373        ));
16374    }
16375
16376    #[test]
16377    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
16378        let supported = json!({
16379            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
16380            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
16381        });
16382        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
16383        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
16384            "workflow_memo_updates": {"supported": false},
16385            "supported_workflow_task_commands": ["upsert_memo"]
16386        }))));
16387        assert!(commands_use_workflow_memo_updates(&[json!({
16388            "type": "upsert_memo",
16389            "entries": {"stage": "processing"}
16390        })]));
16391    }
16392
16393    #[test]
16394    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
16395        let client = Client::new("http://127.0.0.1:8080").expect("client");
16396        let mut worker = Worker::new(client, "rust-workers");
16397        worker.register_workflow("rust.timer", |ctx, _input| async move {
16398            ctx.sleep(Duration::from_secs(5)).await?;
16399            ctx.activity("after-timer", json!([])).await
16400        });
16401
16402        let task = |history_events| WorkflowTask {
16403            task_id: "wft-rust-timer-1".to_string(),
16404            workflow_command_id: None,
16405            workflow_id: Some("wf-rust-timer".to_string()),
16406            run_id: Some("run-rust-timer".to_string()),
16407            workflow_type: "rust.timer".to_string(),
16408            cancel_requested: false,
16409            payload_codec: DEFAULT_CODEC.to_string(),
16410            arguments: Some(
16411                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
16412            ),
16413            history_events,
16414            total_history_events: None,
16415            history_size_bytes: None,
16416            continue_as_new_recommended: None,
16417            history_budget_pressure: None,
16418            next_history_page_token: None,
16419            workflow_task_attempt: 1,
16420            workflow_signal_id: None,
16421            signal_name: None,
16422            signal_arguments: None,
16423            workflow_update_id: None,
16424            update_name: None,
16425            lease_owner: Some("rust-worker".to_string()),
16426        };
16427
16428        let initial = worker
16429            .execute_workflow_task(task(Vec::new()))
16430            .expect("initial timer task");
16431        assert_eq!(
16432            initial,
16433            vec![json!({"type": "start_timer", "delay_seconds": 5})]
16434        );
16435
16436        let activity_result =
16437            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
16438        let replayed = worker
16439            .execute_workflow_task(task(vec![
16440                history_event(
16441                    "TimerScheduled",
16442                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16443                ),
16444                history_event(
16445                    "TimerFired",
16446                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16447                ),
16448                history_event(
16449                    "ActivityCompleted",
16450                    json!({
16451                        "sequence": 2,
16452                        "activity_type": "after-timer",
16453                        "payload_codec": DEFAULT_CODEC,
16454                        "result": activity_result,
16455                    }),
16456                ),
16457            ]))
16458            .expect("replayed workflow task");
16459        assert_eq!(replayed.len(), 1);
16460        assert_eq!(replayed[0]["type"], "complete_workflow");
16461        assert_eq!(
16462            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
16463            json!("done")
16464        );
16465    }
16466
16467    #[test]
16468    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
16469        let client = Client::new("http://127.0.0.1:8080").expect("client");
16470        let mut worker = Worker::new(client, "rust-workers");
16471        worker.register_workflow("rust.continue", |ctx, _input| async move {
16472            ctx.continue_as_new_with_options(
16473                ContinueAsNewOptions::new()
16474                    .workflow_type("rust.next")
16475                    .task_queue("next-workers"),
16476                json!([2, {"cursor": "next"}]),
16477            )
16478        });
16479
16480        let commands = worker
16481            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
16482            .expect("continue-as-new command");
16483
16484        assert_eq!(commands.len(), 1);
16485        assert_eq!(commands[0]["type"], "continue_as_new");
16486        assert_eq!(commands[0]["workflow_type"], "rust.next");
16487        assert_eq!(commands[0]["queue"], "next-workers");
16488        assert_eq!(
16489            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
16490                .expect("continue-as-new arguments"),
16491            json!([2, {"cursor": "next"}])
16492        );
16493    }
16494
16495    #[test]
16496    fn continue_as_new_preserves_typed_arguments() {
16497        let client = Client::new("http://127.0.0.1:8080").expect("client");
16498        let mut worker = Worker::new(client, "rust-workers");
16499        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
16500            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
16501            unreachable!("continue-as-new returns a control-flow error")
16502        });
16503
16504        let commands = worker
16505            .execute_workflow_task(workflow_task(
16506                "rust.typed-continue",
16507                Vec::new(),
16508                DEFAULT_CODEC,
16509            ))
16510            .expect("typed continue-as-new command");
16511
16512        assert_eq!(commands[0]["type"], "continue_as_new");
16513        assert_eq!(
16514            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
16515                .expect("typed continue arguments"),
16516            AvroValue::Array(vec![typed_fidelity_probe()])
16517        );
16518    }
16519
16520    #[test]
16521    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
16522        let client = Client::new("http://127.0.0.1:8080").expect("client");
16523        let mut worker = Worker::new(client, "rust-workers");
16524        worker.register_workflow("rust.continue", |ctx, _input| async move {
16525            ctx.continue_as_new(json!([2]))
16526        });
16527        let task = workflow_task(
16528            "rust.continue",
16529            vec![history_event(
16530                "WorkflowContinuedAsNew",
16531                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
16532            )],
16533            DEFAULT_CODEC,
16534        );
16535
16536        for _worker_restart_or_redelivery in 0..2 {
16537            let commands = worker
16538                .execute_workflow_task(task.clone())
16539                .expect("recorded transition replays");
16540            assert!(
16541                commands.is_empty(),
16542                "replay must not emit another successor"
16543            );
16544        }
16545    }
16546
16547    #[test]
16548    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
16549        let ctx = workflow_context(Vec::new());
16550        let error = ctx
16551            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
16552            .expect_err("blank queue must be rejected");
16553
16554        let Error::InvalidContinueAsNewOptions(error) = error else {
16555            panic!("expected typed continue-as-new validation error");
16556        };
16557        assert_eq!(error.field, "task_queue");
16558        assert!(ctx.take_commands().expect("commands").is_empty());
16559    }
16560
16561    #[test]
16562    fn workflow_context_exposes_server_history_budget() {
16563        let client = Client::new("http://127.0.0.1:8080").expect("client");
16564        let mut worker = Worker::new(client, "rust-workers");
16565        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
16566            let budget = ctx.history_budget()?;
16567            Ok(json!({
16568                "events": budget.event_count,
16569                "bytes": budget.size_bytes,
16570                "recommended": budget.continue_as_new_recommended,
16571                "pressure": budget.pressure,
16572            }))
16573        });
16574        let task: WorkflowTask = serde_json::from_value(json!({
16575            "task_id": "task-history-budget",
16576            "workflow_type": "rust.history-budget",
16577            "payload_codec": DEFAULT_CODEC,
16578            "history_events": [],
16579            "total_history_events": 480,
16580            "history_size_bytes": 1_048_576,
16581            "continue_as_new_recommended": true,
16582            "history_budget_pressure": "continue_as_new_recommended",
16583        }))
16584        .expect("published workflow task");
16585
16586        let commands = worker
16587            .execute_workflow_task(task)
16588            .expect("history-budget workflow");
16589        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
16590        assert_eq!(result["events"], 480);
16591        assert_eq!(result["bytes"], 1_048_576);
16592        assert_eq!(result["recommended"], true);
16593        assert_eq!(result["pressure"], "continue_as_new_recommended");
16594    }
16595
16596    #[test]
16597    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
16598        let client = Client::new("http://127.0.0.1:8080").expect("client");
16599        let mut worker = Worker::new(client, "rust-workers");
16600        worker.register_workflow("rust.failing", |_ctx, _input| async move {
16601            Err(Error::Codec("rust_conformance_failure".to_string()))
16602        });
16603        let task = WorkflowTask {
16604            task_id: "wft-rust-failing-1".to_string(),
16605            workflow_command_id: None,
16606            workflow_id: Some("wf-rust-failing".to_string()),
16607            run_id: Some("run-rust-failing".to_string()),
16608            workflow_type: "rust.failing".to_string(),
16609            cancel_requested: false,
16610            payload_codec: DEFAULT_CODEC.to_string(),
16611            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
16612            history_events: Vec::new(),
16613            total_history_events: Some(0),
16614            history_size_bytes: None,
16615            continue_as_new_recommended: None,
16616            history_budget_pressure: None,
16617            next_history_page_token: None,
16618            workflow_task_attempt: 1,
16619            workflow_signal_id: None,
16620            signal_name: None,
16621            signal_arguments: None,
16622            workflow_update_id: None,
16623            update_name: None,
16624            lease_owner: Some("rust-worker".to_string()),
16625        };
16626
16627        let commands = worker
16628            .execute_workflow_task(task)
16629            .expect("handler failure becomes a workflow command");
16630
16631        assert_eq!(commands.len(), 1);
16632        assert_eq!(commands[0]["type"], "fail_workflow");
16633        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
16634        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
16635        assert_eq!(commands[0]["non_retryable"], false);
16636        assert_eq!(
16637            commands[0]["message"],
16638            "codec error: rust_conformance_failure"
16639        );
16640        assert_eq!(
16641            commands[0]["exception"]["message"],
16642            "codec error: rust_conformance_failure"
16643        );
16644    }
16645
16646    #[test]
16647    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
16648        let client = Client::new("http://127.0.0.1:8080").expect("client");
16649        let mut worker = Worker::new(client, "rust-workers");
16650        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
16651            let _: String = ctx.side_effect(|| "captured".to_string())?;
16652            Err(Error::WorkerLoop("application failure".to_string()))
16653        });
16654
16655        let commands = worker
16656            .execute_workflow_task(workflow_task(
16657                "rust.failing-after-side-effect",
16658                Vec::new(),
16659                DEFAULT_CODEC,
16660            ))
16661            .expect("ordinary failure remains a workflow decision");
16662
16663        assert_eq!(commands.len(), 2);
16664        assert_eq!(commands[0]["type"], "record_side_effect");
16665        assert_eq!(commands[1]["type"], "fail_workflow");
16666    }
16667
16668    #[test]
16669    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
16670        let client = Client::new("http://127.0.0.1:8080").expect("client");
16671        let mut worker = Worker::new(client, "rust-workers");
16672        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
16673            Err(Error::WorkerLoop("application failure".to_string()))
16674        });
16675        let result =
16676            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
16677
16678        let error = worker
16679            .execute_workflow_task(workflow_task(
16680                "rust.removed-side-effect",
16681                vec![history_event(
16682                    "SideEffectRecorded",
16683                    json!({"sequence": 1, "result": result}),
16684                )],
16685                DEFAULT_CODEC,
16686            ))
16687            .expect_err("removed committed history must not become fail_workflow");
16688
16689        let Error::NonDeterministicReplay(failure) = error else {
16690            panic!("expected typed replay failure");
16691        };
16692        assert_eq!(failure.reason, "recorded_commands_unconsumed");
16693        assert_eq!(failure.sequence, Some(1));
16694        assert_eq!(failure.expected.as_deref(), Some("side effect"));
16695    }
16696
16697    #[test]
16698    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
16699        let client = Client::new("http://127.0.0.1:8080").expect("client");
16700        let mut worker = Worker::new(client, "rust-workers");
16701        worker.register_workflow(
16702            "rust.side-effect-before-marker-error",
16703            |ctx, _input| async move {
16704                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
16705                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
16706                ctx.get_version("restart-safe", 2, 2)?;
16707                Ok(Value::Null)
16708            },
16709        );
16710
16711        let error = worker
16712            .execute_workflow_task(workflow_task(
16713                "rust.side-effect-before-marker-error",
16714                vec![history_event(
16715                    "VersionMarkerRecorded",
16716                    json!({
16717                        "sequence": 1,
16718                        "change_id": "restart-safe",
16719                        "version": 1,
16720                        "min_supported": 1,
16721                        "max_supported": 1,
16722                    }),
16723                )],
16724                DEFAULT_CODEC,
16725            ))
16726            .expect_err("replay error must return no queued workflow commands");
16727
16728        let Error::NonDeterministicReplay(failure) = error else {
16729            panic!("expected typed replay failure");
16730        };
16731        assert_eq!(failure.reason, "version_marker_incompatible_range");
16732        assert_eq!(failure.sequence, Some(1));
16733    }
16734
16735    #[test]
16736    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
16737        let client = Client::new("http://127.0.0.1:8080").expect("client");
16738        let mut worker = Worker::new(client, "rust-workers");
16739        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
16740            ctx.sleep(Duration::from_secs(5)).await?;
16741            Ok(json!({"status": "timer fired"}))
16742        });
16743
16744        let task = WorkflowTask {
16745            task_id: "wft-rust-timer-pending".to_string(),
16746            workflow_command_id: None,
16747            workflow_id: Some("wf-rust-timer".to_string()),
16748            run_id: Some("run-rust-timer".to_string()),
16749            workflow_type: "rust.timer.pending".to_string(),
16750            cancel_requested: false,
16751            payload_codec: DEFAULT_CODEC.to_string(),
16752            arguments: Some(
16753                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
16754            ),
16755            history_events: vec![history_event(
16756                "TimerScheduled",
16757                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16758            )],
16759            total_history_events: Some(1),
16760            history_size_bytes: None,
16761            continue_as_new_recommended: None,
16762            history_budget_pressure: None,
16763            next_history_page_token: None,
16764            workflow_task_attempt: 1,
16765            workflow_signal_id: None,
16766            signal_name: None,
16767            signal_arguments: None,
16768            workflow_update_id: None,
16769            update_name: None,
16770            lease_owner: Some("rust-worker".to_string()),
16771        };
16772
16773        for _redelivery_or_restart in 0..2 {
16774            let commands = worker
16775                .execute_workflow_task(task.clone())
16776                .expect("recorded timer remains pending");
16777            assert!(
16778                commands.is_empty(),
16779                "recorded timer must not be rescheduled"
16780            );
16781        }
16782    }
16783
16784    #[test]
16785    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
16786        let client = Client::new("http://127.0.0.1:8080").expect("client");
16787        let mut worker = Worker::new(client, "rust-workers");
16788        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
16789            Ok(json!({"status": "completed"}))
16790        });
16791        let task = WorkflowTask {
16792            task_id: "wft-rust-timer-removed".to_string(),
16793            workflow_command_id: None,
16794            workflow_id: Some("wf-rust-timer".to_string()),
16795            run_id: Some("run-rust-timer".to_string()),
16796            workflow_type: "rust.timer.removed".to_string(),
16797            cancel_requested: false,
16798            payload_codec: DEFAULT_CODEC.to_string(),
16799            arguments: Some(
16800                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
16801            ),
16802            history_events: vec![
16803                history_event(
16804                    "TimerScheduled",
16805                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16806                ),
16807                history_event(
16808                    "TimerFired",
16809                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
16810                ),
16811            ],
16812            total_history_events: Some(2),
16813            history_size_bytes: None,
16814            continue_as_new_recommended: None,
16815            history_budget_pressure: None,
16816            next_history_page_token: None,
16817            workflow_task_attempt: 1,
16818            workflow_signal_id: None,
16819            signal_name: None,
16820            signal_arguments: None,
16821            workflow_update_id: None,
16822            update_name: None,
16823            lease_owner: Some("rust-worker".to_string()),
16824        };
16825
16826        let Error::NonDeterministicReplay(failure) = worker
16827            .execute_workflow_task(task)
16828            .expect_err("removed timer must fail replay")
16829        else {
16830            panic!("expected typed replay failure");
16831        };
16832        assert_eq!(failure.reason, "recorded_commands_unconsumed");
16833        assert_eq!(failure.sequence, Some(1));
16834    }
16835
16836    #[test]
16837    fn workflow_context_emits_explicit_child_workflow_contract() {
16838        let ctx = WorkflowContext {
16839            state: Arc::new(Mutex::new(
16840                WorkflowState::new_with_identity(
16841                    Vec::new(),
16842                    Some("wf-parent".to_string()),
16843                    Some("run-parent".to_string()),
16844                    "parent-workers".to_string(),
16845                    DEFAULT_CODEC.to_string(),
16846                    None,
16847                )
16848                .expect("workflow state"),
16849            )),
16850        };
16851        let options = ChildWorkflowOptions::new("python-workers")
16852            .parent_close_policy(ParentClosePolicy::RequestCancel)
16853            .retry_policy(ChildWorkflowRetryPolicy {
16854                max_attempts: Some(3),
16855                backoff_seconds: vec![1, 5],
16856                non_retryable_error_types: vec!["ValidationError".to_string()],
16857            })
16858            .execution_timeout_seconds(600)
16859            .run_timeout_seconds(120);
16860        let mut call = Box::pin(ctx.start_child_workflow(
16861            "python.fulfil-order",
16862            options,
16863            json!([{"order_id": "order-42"}]),
16864        ));
16865        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16866
16867        assert!(matches!(
16868            call.as_mut().poll(&mut task_context),
16869            Poll::Pending
16870        ));
16871        let commands = ctx.take_commands().expect("commands");
16872        assert_eq!(commands.len(), 1);
16873        let command = &commands[0];
16874        assert_eq!(command["type"], "start_child_workflow");
16875        assert_eq!(command["workflow_type"], "python.fulfil-order");
16876        assert_eq!(command["queue"], "python-workers");
16877        assert_eq!(command["parent_close_policy"], "request_cancel");
16878        assert_eq!(command["retry_policy"]["max_attempts"], 3);
16879        assert_eq!(command["execution_timeout_seconds"], 600);
16880        assert_eq!(command["run_timeout_seconds"], 120);
16881        assert_eq!(
16882            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
16883            json!([{"order_id": "order-42"}])
16884        );
16885    }
16886
16887    fn child_parent_worker() -> Worker {
16888        let client = Client::new("http://127.0.0.1:8080").expect("client");
16889        let mut worker = Worker::new(client, "rust-parent-workers");
16890        worker.register_workflow("rust.parent", |ctx, _input| async move {
16891            let child = ctx
16892                .start_child_workflow(
16893                    "python.child",
16894                    ChildWorkflowOptions::new("python-child-workers")
16895                        .parent_close_policy(ParentClosePolicy::Terminate),
16896                    json!([{"codec_probe": [1, true, "rust"]}]),
16897                )
16898                .await?;
16899            Ok(json!({
16900                "parent_workflow_id": child.parent.workflow_id,
16901                "parent_run_id": child.parent.run_id,
16902                "child_workflow_id": child.child.workflow_id,
16903                "child_run_id": child.child.run_id,
16904                "child_workflow_type": child.child_workflow_type,
16905                "result": child.result,
16906            }))
16907        });
16908        worker
16909    }
16910
16911    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
16912        WorkflowTask {
16913            task_id: "wft-child-parent".to_string(),
16914            workflow_command_id: None,
16915            workflow_id: Some("wf-parent".to_string()),
16916            run_id: Some("run-parent".to_string()),
16917            workflow_type: "rust.parent".to_string(),
16918            cancel_requested: false,
16919            payload_codec: DEFAULT_CODEC.to_string(),
16920            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
16921            history_events: vec![
16922                HistoryEvent {
16923                    event_type: "ChildWorkflowScheduled".to_string(),
16924                    payload: json!({
16925                        "sequence": 1,
16926                        "child_call_id": "call-child",
16927                        "child_workflow_instance_id": "wf-child",
16928                        "child_workflow_run_id": "run-child",
16929                        "child_workflow_type": "python.child",
16930                    }),
16931                    raw: HashMap::new(),
16932                },
16933                HistoryEvent {
16934                    event_type: event_type.to_string(),
16935                    payload,
16936                    raw: HashMap::new(),
16937                },
16938            ],
16939            total_history_events: Some(2),
16940            history_size_bytes: None,
16941            continue_as_new_recommended: None,
16942            history_budget_pressure: None,
16943            next_history_page_token: None,
16944            workflow_task_attempt: 1,
16945            workflow_signal_id: None,
16946            signal_name: None,
16947            signal_arguments: None,
16948            workflow_update_id: None,
16949            update_name: None,
16950            lease_owner: Some("rust-worker".to_string()),
16951        }
16952    }
16953
16954    #[test]
16955    fn committed_child_result_replays_without_starting_a_duplicate() {
16956        let worker = child_parent_worker();
16957        let task = child_parent_task(
16958            "ChildRunCompleted",
16959            json!({
16960                "sequence": 1,
16961                "child_call_id": "call-child",
16962                "child_workflow_instance_id": "wf-child",
16963                "child_workflow_run_id": "run-child",
16964                "child_workflow_type": "python.child",
16965                "payload_codec": DEFAULT_CODEC,
16966                "result": fixture_envelope(json!({"from":"python","ok":true})),
16967            }),
16968        );
16969
16970        for _restart in 0..2 {
16971            let commands = worker
16972                .execute_workflow_task(task.clone())
16973                .expect("replayed parent task");
16974            assert_eq!(commands.len(), 1);
16975            assert_eq!(commands[0]["type"], "complete_workflow");
16976            assert!(!commands
16977                .iter()
16978                .any(|command| command["type"] == "start_child_workflow"));
16979            let output =
16980                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
16981            assert_eq!(output["parent_workflow_id"], "wf-parent");
16982            assert_eq!(output["parent_run_id"], "run-parent");
16983            assert_eq!(output["child_workflow_id"], "wf-child");
16984            assert_eq!(output["child_run_id"], "run-child");
16985            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
16986        }
16987    }
16988
16989    #[test]
16990    fn typed_child_arguments_and_results_survive_replay() {
16991        let client = Client::new("http://127.0.0.1:8080").expect("client");
16992        let mut worker = Worker::new(client, "rust-parent-workers");
16993        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
16994            let child = ctx
16995                .start_child_workflow_avro_value(
16996                    "python.typed-child",
16997                    ChildWorkflowOptions::new("python-workers"),
16998                    AvroValue::Array(vec![typed_fidelity_probe()]),
16999                )
17000                .await?;
17001            Ok(child.result)
17002        });
17003
17004        let initial = worker
17005            .execute_workflow_task(workflow_task(
17006                "rust.typed-parent",
17007                Vec::new(),
17008                DEFAULT_CODEC,
17009            ))
17010            .expect("typed child start");
17011        assert_eq!(initial[0]["type"], "start_child_workflow");
17012        assert_eq!(
17013            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
17014                .expect("typed child arguments"),
17015            AvroValue::Array(vec![typed_fidelity_probe()])
17016        );
17017
17018        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
17019            .expect("typed child result");
17020        let task = workflow_task(
17021            "rust.typed-parent",
17022            vec![
17023                history_event(
17024                    "ChildWorkflowScheduled",
17025                    json!({
17026                        "sequence": 1,
17027                        "child_call_id": "call-typed",
17028                        "child_workflow_instance_id": "wf-child",
17029                        "child_workflow_run_id": "run-child",
17030                        "child_workflow_type": "python.typed-child",
17031                    }),
17032                ),
17033                history_event(
17034                    "ChildRunCompleted",
17035                    json!({
17036                        "sequence": 1,
17037                        "child_call_id": "call-typed",
17038                        "child_workflow_instance_id": "wf-child",
17039                        "child_workflow_run_id": "run-child",
17040                        "child_workflow_type": "python.typed-child",
17041                        "payload_codec": DEFAULT_CODEC,
17042                        "result": result,
17043                    }),
17044                ),
17045            ],
17046            DEFAULT_CODEC,
17047        );
17048
17049        let commands = worker
17050            .execute_workflow_task(task)
17051            .expect("typed child replay");
17052        assert_eq!(commands[0]["type"], "complete_workflow");
17053        assert_eq!(
17054            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17055                .expect("typed parent result"),
17056            typed_fidelity_probe()
17057        );
17058    }
17059
17060    #[test]
17061    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
17062        let worker = child_parent_worker();
17063        let mut task = child_parent_task("unused", Value::Null);
17064        task.history_events.truncate(1);
17065        task.total_history_events = Some(1);
17066
17067        for _redelivery_or_restart in 0..2 {
17068            let commands = worker
17069                .execute_workflow_task(task.clone())
17070                .expect("recorded child remains pending");
17071            assert!(
17072                commands.is_empty(),
17073                "recorded pending child must not be started again"
17074            );
17075        }
17076    }
17077
17078    #[test]
17079    fn child_cancellation_becomes_stable_parent_failure_command() {
17080        let worker = child_parent_worker();
17081        let task = child_parent_task(
17082            "ChildRunCancelled",
17083            json!({
17084                "sequence": 1,
17085                "child_workflow_instance_id": "wf-child",
17086                "child_workflow_run_id": "run-child",
17087                "child_workflow_type": "python.child",
17088                "failure_id": "failure-child",
17089                "failure_category": "cancelled",
17090                "message": "cancelled by parent-close policy",
17091            }),
17092        );
17093
17094        let commands = worker
17095            .execute_workflow_task(task)
17096            .expect("parent settlement");
17097        assert_eq!(commands.len(), 1);
17098        assert_eq!(commands[0]["type"], "fail_workflow");
17099        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
17100        assert_eq!(
17101            commands[0]["exception"]["properties"]["reason"],
17102            "cancelled"
17103        );
17104        assert_eq!(
17105            commands[0]["exception"]["properties"]["child_workflow_run_id"],
17106            "run-child"
17107        );
17108    }
17109
17110    #[test]
17111    fn workflow_can_handle_typed_child_failure() {
17112        let client = Client::new("http://127.0.0.1:8080").expect("client");
17113        let mut worker = Worker::new(client, "rust-parent-workers");
17114        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
17115            match ctx
17116                .start_child_workflow(
17117                    "python.child",
17118                    ChildWorkflowOptions::new("python-child-workers"),
17119                    json!([]),
17120                )
17121                .await
17122            {
17123                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
17124                    "reason": failure.reason,
17125                    "failure_id": failure.failure_id,
17126                    "exception_class": failure.exception_class,
17127                    "child_run_id": failure.child_workflow_run_id,
17128                })),
17129                Err(error) => Err(error),
17130                Ok(_) => Err(Error::WorkerLoop(
17131                    "child unexpectedly succeeded".to_string(),
17132                )),
17133            }
17134        });
17135        let mut task = child_parent_task(
17136            "ChildRunFailed",
17137            json!({
17138                "sequence": 1,
17139                "child_workflow_instance_id": "wf-child",
17140                "child_workflow_run_id": "run-child",
17141                "child_workflow_type": "python.child",
17142                "failure_id": "failure-child",
17143                "failure_category": "child_workflow",
17144                "message": "payment rejected",
17145                "exception": {
17146                    "type": "PaymentRejected",
17147                    "class": "payments.PaymentRejected",
17148                    "message": "payment rejected"
17149                }
17150            }),
17151        );
17152        task.workflow_type = "rust.handled-parent".to_string();
17153
17154        let commands = worker.execute_workflow_task(task).expect("handled failure");
17155        assert_eq!(commands[0]["type"], "complete_workflow");
17156        let output =
17157            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
17158        assert_eq!(output["reason"], "child_workflow");
17159        assert_eq!(output["failure_id"], "failure-child");
17160        assert_eq!(output["exception_class"], "payments.PaymentRejected");
17161        assert_eq!(output["child_run_id"], "run-child");
17162    }
17163
17164    #[test]
17165    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
17166        let client = Client::new("http://127.0.0.1:8080").expect("client");
17167        let mut worker = Worker::new(client, "rust-workers");
17168
17169        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
17170            let signal = ctx.wait_signal("start").await?;
17171            let name = signal
17172                .first()
17173                .and_then(|value| value.as_str())
17174                .unwrap_or("world");
17175            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
17176            Ok(json!({
17177                "greeting": greeting,
17178                "language": "rust"
17179            }))
17180        });
17181
17182        let signal_arguments =
17183            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
17184        let task = WorkflowTask {
17185            task_id: "wft-rust-signal-1".to_string(),
17186            workflow_command_id: None,
17187            workflow_id: Some("wf-rust-hello".to_string()),
17188            run_id: Some("run-rust-hello".to_string()),
17189            workflow_type: "rust.hello_workflow".to_string(),
17190            cancel_requested: false,
17191            payload_codec: DEFAULT_CODEC.to_string(),
17192            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
17193            history_events: vec![HistoryEvent {
17194                event_type: "SignalReceived".to_string(),
17195                payload: json!({
17196                    "signal_id": "sig-rust-1",
17197                    "signal_name": "start"
17198                }),
17199                raw: HashMap::new(),
17200            }],
17201            total_history_events: Some(1),
17202            history_size_bytes: None,
17203            continue_as_new_recommended: None,
17204            history_budget_pressure: None,
17205            next_history_page_token: None,
17206            workflow_task_attempt: 1,
17207            workflow_signal_id: Some("sig-rust-1".to_string()),
17208            signal_name: Some("start".to_string()),
17209            signal_arguments: Some(signal_arguments),
17210            workflow_update_id: None,
17211            update_name: None,
17212            lease_owner: Some("rust-worker".to_string()),
17213        };
17214
17215        let commands = worker.execute_workflow_task(task).expect("workflow task");
17216
17217        assert_eq!(commands.len(), 1);
17218        assert_eq!(commands[0]["type"], "schedule_activity");
17219        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
17220        assert_eq!(
17221            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
17222            json!(["Rust"])
17223        );
17224    }
17225
17226    #[test]
17227    fn workflow_task_appends_paginated_history_events() {
17228        let mut task = WorkflowTask {
17229            task_id: "wft-rust-pages-1".to_string(),
17230            workflow_command_id: None,
17231            workflow_id: Some("wf-rust-pages".to_string()),
17232            run_id: Some("run-rust-pages".to_string()),
17233            workflow_type: "rust.hello_workflow".to_string(),
17234            cancel_requested: false,
17235            payload_codec: DEFAULT_CODEC.to_string(),
17236            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
17237            history_events: vec![HistoryEvent {
17238                event_type: "WorkflowStarted".to_string(),
17239                payload: json!({}),
17240                raw: HashMap::new(),
17241            }],
17242            total_history_events: Some(3),
17243            history_size_bytes: None,
17244            continue_as_new_recommended: None,
17245            history_budget_pressure: None,
17246            next_history_page_token: Some("MQ==".to_string()),
17247            workflow_task_attempt: 1,
17248            workflow_signal_id: None,
17249            signal_name: None,
17250            signal_arguments: None,
17251            workflow_update_id: None,
17252            update_name: None,
17253            lease_owner: Some("rust-worker".to_string()),
17254        };
17255
17256        task.append_history_page(WorkflowTaskHistoryPage {
17257            history_events: vec![
17258                HistoryEvent {
17259                    event_type: "SignalReceived".to_string(),
17260                    payload: json!({
17261                        "signal_id": "sig-rust-1",
17262                        "signal_name": "start",
17263                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
17264                            .expect("signal arguments")
17265                    }),
17266                    raw: HashMap::new(),
17267                },
17268                HistoryEvent {
17269                    event_type: "MarkerRecorded".to_string(),
17270                    payload: json!({"sequence": 3}),
17271                    raw: HashMap::new(),
17272                },
17273            ],
17274            total_history_events: Some(3),
17275            next_history_page_token: None,
17276        });
17277
17278        assert_eq!(task.history_events.len(), 3);
17279        assert_eq!(task.total_history_events, Some(3));
17280        assert_eq!(task.next_history_page_token, None);
17281
17282        let signal = task
17283            .history_events
17284            .iter()
17285            .find(|event| event.event_type == "SignalReceived")
17286            .expect("signal event");
17287        assert_eq!(
17288            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
17289            vec![AvroValue::String("Rust".to_string())]
17290        );
17291    }
17292
17293    #[tokio::test]
17294    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
17295        let client = Client::new("http://127.0.0.1:8080").expect("client");
17296        let mut worker = Worker::new(client, "rust-workers");
17297        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
17298        worker.register_query("counter", "current", |ctx, _args| async move {
17299            let mut count = 0_i64;
17300            for signal in ctx.signal_events() {
17301                let value = signal
17302                    .arguments
17303                    .first()
17304                    .and_then(Value::as_i64)
17305                    .unwrap_or_default();
17306                match signal.name.as_str() {
17307                    "increment" => count += value,
17308                    "set" => count = value,
17309                    _ => {}
17310                }
17311            }
17312            Ok(json!(count))
17313        });
17314
17315        let task = QueryTask {
17316            query_task_id: "query-rust-counter".to_string(),
17317            query_task_attempt: 1,
17318            lease_owner: Some("rust-worker".to_string()),
17319            workflow_id: Some("counter-1".to_string()),
17320            run_id: Some("run-counter-1".to_string()),
17321            workflow_type: "counter".to_string(),
17322            query_name: "current".to_string(),
17323            payload_codec: DEFAULT_CODEC.to_string(),
17324            workflow_arguments: Some(
17325                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
17326            ),
17327            query_arguments: Some(
17328                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
17329            ),
17330            history_events: vec![
17331                HistoryEvent {
17332                    event_type: "SignalReceived".to_string(),
17333                    payload: json!({
17334                        "signal_id": "php-signal-1",
17335                        "signal_name": "increment",
17336                        "workflow_sequence": 1,
17337                        "payload_codec": DEFAULT_CODEC,
17338                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
17339                    }),
17340                    raw: HashMap::new(),
17341                },
17342                HistoryEvent {
17343                    event_type: "SignalReceived".to_string(),
17344                    payload: json!({
17345                        "signal_id": "python-signal-2",
17346                        "signal_name": "increment",
17347                        "workflow_sequence": 2,
17348                        "payload_codec": DEFAULT_CODEC,
17349                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
17350                    }),
17351                    raw: HashMap::new(),
17352                },
17353                HistoryEvent {
17354                    event_type: "SignalReceived".to_string(),
17355                    payload: json!({
17356                        "signal_id": "rust-signal-3",
17357                        "signal_name": "set",
17358                        "workflow_sequence": 3,
17359                        "payload_codec": DEFAULT_CODEC,
17360                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
17361                    }),
17362                    raw: HashMap::new(),
17363                },
17364            ],
17365            history_export: None,
17366            run_status: Some("completed".to_string()),
17367        };
17368
17369        let result = worker.execute_query_task(task).await.expect("query result");
17370        assert_eq!(result.into_json().expect("query projection"), json!(0));
17371    }
17372
17373    #[tokio::test]
17374    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
17375        let worker = replay_counter_worker();
17376        let running_history = json!([
17377            {
17378                "type": "ActivityCompleted",
17379                "payload": {
17380                    "sequence": 1,
17381                    "activity_type": "load-counter",
17382                    "payload_codec": DEFAULT_CODEC,
17383                    "result": fixture_envelope(json!("loaded"))
17384                }
17385            },
17386            {
17387                "type": "SignalWaitOpened",
17388                "payload": {
17389                    "sequence": 3,
17390                    "signal_name": "increment"
17391                }
17392            },
17393            {
17394                "type": "SignalReceived",
17395                "payload": {
17396                    "signal_id": "signal-3",
17397                    "signal_name": "increment",
17398                    "workflow_sequence": 2,
17399                    "payload_codec": DEFAULT_CODEC,
17400                    "arguments": fixture_envelope(json!([3]))
17401                }
17402            },
17403            {
17404                "type": "SignalApplied",
17405                "payload": {
17406                    "sequence": 3,
17407                    "signal_id": "signal-3",
17408                    "signal_name": "increment",
17409                    "payload_codec": DEFAULT_CODEC,
17410                    "value": fixture_envelope(json!([3]))
17411                }
17412            }
17413        ]);
17414
17415        let running = worker
17416            .execute_query_task(replay_counter_query(
17417                "current",
17418                running_history.clone(),
17419                "running",
17420            ))
17421            .await
17422            .expect("running replay query");
17423        assert_eq!(
17424            running.clone().into_json().expect("query projection"),
17425            json!({"loaded": "loaded", "count": 3, "finished": false})
17426        );
17427
17428        let detached = worker
17429            .execute_query_task(replay_counter_query(
17430                "detached-mutation",
17431                running_history.clone(),
17432                "running",
17433            ))
17434            .await
17435            .expect("query mutates only its detached state clone");
17436        assert_eq!(detached.into_json().expect("query projection"), json!(999));
17437        let failed = worker
17438            .execute_query_task(replay_counter_query(
17439                "failed-mutation",
17440                running_history.clone(),
17441                "running",
17442            ))
17443            .await
17444            .expect_err("failed query");
17445        assert_eq!(failed.reason, "query_rejected");
17446        let unchanged = worker
17447            .execute_query_task(replay_counter_query("current", running_history, "running"))
17448            .await
17449            .expect("later query reconstructs unchanged state");
17450        assert_eq!(unchanged, running);
17451
17452        let restarted_worker = replay_counter_worker();
17453        let empty_arguments = fixture_envelope(json!([]));
17454        let loaded_result = fixture_envelope(json!("loaded"));
17455        let signal_three = fixture_blob(json!([3]));
17456        let signal_five = fixture_blob(json!([5]));
17457        let restarted_task: QueryTask = serde_json::from_value(json!({
17458            "query_task_id": "query-after-restart",
17459            "workflow_id": "counter-1",
17460            "run_id": "run-counter-1",
17461            "workflow_type": "replay-counter",
17462            "query_name": "current",
17463            "payload_codec": DEFAULT_CODEC,
17464            "workflow_arguments": empty_arguments.clone(),
17465            "query_arguments": empty_arguments,
17466            "history_events": [],
17467            "history_export": {
17468                "payloads": {"codec": DEFAULT_CODEC},
17469                "history_events": [
17470                    {
17471                        "type": "ActivityCompleted",
17472                        "payload": {
17473                            "sequence": 1,
17474                            "activity_type": "load-counter",
17475                            "payload_codec": DEFAULT_CODEC,
17476                            "result": null
17477                        }
17478                    },
17479                    {
17480                        "type": "SignalWaitOpened",
17481                        "payload": {
17482                            "sequence": 3,
17483                            "signal_name": "increment"
17484                        }
17485                    },
17486                    {
17487                        "type": "SignalReceived",
17488                        "payload": {
17489                            "signal_id": "signal-3",
17490                            "signal_name": "increment",
17491                            "workflow_sequence": 2
17492                        }
17493                    },
17494                    {
17495                        "type": "SignalApplied",
17496                        "payload": {
17497                            "sequence": 3,
17498                            "signal_id": "signal-3",
17499                            "signal_name": "increment"
17500                        }
17501                    },
17502                    {
17503                        "type": "SignalWaitOpened",
17504                        "payload": {
17505                            "sequence": 5,
17506                            "signal_name": "increment"
17507                        }
17508                    },
17509                    {
17510                        "type": "SignalReceived",
17511                        "payload": {
17512                            "signal_id": "signal-5",
17513                            "signal_name": "increment",
17514                            "workflow_sequence": 4
17515                        }
17516                    },
17517                    {
17518                        "type": "SignalApplied",
17519                        "payload": {
17520                            "sequence": 5,
17521                            "signal_id": "signal-5",
17522                            "signal_name": "increment"
17523                        }
17524                    }
17525                ],
17526                "activities": [{
17527                    "sequence": 1,
17528                    "activity_type": "load-counter",
17529                    "payload_codec": DEFAULT_CODEC,
17530                    "result": loaded_result
17531                }],
17532                "signals": [
17533                    {
17534                        "id": "signal-3",
17535                        "name": "increment",
17536                        "workflow_sequence": 2,
17537                        "payload_codec": DEFAULT_CODEC,
17538                        "arguments": signal_three
17539                    },
17540                    {
17541                        "id": "signal-5",
17542                        "name": "increment",
17543                        "workflow_sequence": 4,
17544                        "payload_codec": DEFAULT_CODEC,
17545                        "arguments": signal_five
17546                    }
17547                ]
17548            },
17549            "run_status": "completed"
17550        }))
17551        .expect("cold replay query task");
17552        let completed = restarted_worker
17553            .execute_query_task(restarted_task)
17554            .await
17555            .expect("completed cold replay query");
17556        assert_eq!(
17557            completed.into_json().expect("query projection"),
17558            json!({"loaded": "loaded", "count": 8, "finished": true})
17559        );
17560    }
17561
17562    #[tokio::test]
17563    async fn replayed_query_replay_failures_are_machine_readable() {
17564        let worker = replay_counter_worker();
17565        let task = replay_counter_query(
17566            "current",
17567            json!([{
17568                "type": "ActivityCompleted",
17569                "payload": {
17570                    "sequence": 1,
17571                    "payload_codec": DEFAULT_CODEC,
17572                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
17573                }
17574            }]),
17575            "running",
17576        );
17577        let failure = worker
17578            .execute_query_task(task)
17579            .await
17580            .expect_err("invalid replay history payload");
17581        assert_eq!(failure.reason, "query_payload_decode_failed");
17582        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
17583        assert!(failure.message.contains("invalid_payload_framing"));
17584    }
17585
17586    #[tokio::test]
17587    async fn query_task_restores_compact_history_from_export() {
17588        let client = Client::new("http://127.0.0.1:8080").expect("client");
17589        let mut worker = Worker::new(client, "rust-workers");
17590        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
17591        worker.register_query("counter", "current", |ctx, _args| async move {
17592            Ok(json!(ctx.signals("increment")[0][0]))
17593        });
17594        let empty_arguments = fixture_envelope(json!([]));
17595        let exported_signal = fixture_blob(json!([9]));
17596        let task: QueryTask = serde_json::from_value(json!({
17597            "query_task_id": "query-export",
17598            "workflow_type": "counter",
17599            "query_name": "current",
17600            "payload_codec": DEFAULT_CODEC,
17601            "workflow_arguments": empty_arguments.clone(),
17602            "query_arguments": empty_arguments,
17603            "history_events": [],
17604            "history_export": {
17605                "payloads": {"codec": DEFAULT_CODEC},
17606                "history_events": [{
17607                    "type": "SignalReceived",
17608                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
17609                }],
17610                "signals": [{
17611                    "id": "signal-export",
17612                    "name": "increment",
17613                    "status": "applied",
17614                    "workflow_sequence": 1,
17615                    "payload_codec": DEFAULT_CODEC,
17616                    "arguments": exported_signal
17617                }]
17618            }
17619        }))
17620        .expect("query task");
17621
17622        let result = worker.execute_query_task(task).await.expect("query result");
17623        assert_eq!(result.into_json().expect("query projection"), json!(9));
17624    }
17625
17626    #[tokio::test]
17627    async fn query_task_failures_have_stable_reasons() {
17628        let client = Client::new("http://127.0.0.1:8080").expect("client");
17629        let mut worker = Worker::new(client, "rust-workers");
17630        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
17631        worker.register_query(
17632            "counter",
17633            "current",
17634            |_ctx, _args| async move { Ok(json!(0)) },
17635        );
17636
17637        let base_task = QueryTask {
17638            query_task_id: "query-errors".to_string(),
17639            query_task_attempt: 1,
17640            lease_owner: None,
17641            workflow_id: Some("counter-errors".to_string()),
17642            run_id: Some("run-errors".to_string()),
17643            workflow_type: "counter".to_string(),
17644            query_name: "missing".to_string(),
17645            payload_codec: DEFAULT_CODEC.to_string(),
17646            workflow_arguments: Some(fixture_envelope(json!([]))),
17647            query_arguments: Some(fixture_envelope(json!([]))),
17648            history_events: Vec::new(),
17649            history_export: None,
17650            run_status: Some("running".to_string()),
17651        };
17652
17653        let unknown = worker
17654            .execute_query_task(base_task.clone())
17655            .await
17656            .expect_err("unknown query");
17657        assert_eq!(unknown.reason, "rejected_unknown_query");
17658
17659        let mut malformed = base_task;
17660        malformed.query_name = "current".to_string();
17661        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
17662        let malformed = worker
17663            .execute_query_task(malformed)
17664            .await
17665            .expect_err("malformed payload");
17666        assert_eq!(malformed.reason, "query_payload_decode_failed");
17667
17668        let client = Client::new("http://127.0.0.1:8080").expect("client");
17669        let mut unavailable_worker = Worker::new(client, "rust-workers");
17670        unavailable_worker
17671            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
17672        let empty_arguments = fixture_envelope(json!([]));
17673        let unavailable_task: QueryTask = serde_json::from_value(json!({
17674            "query_task_id": "query-unavailable",
17675            "workflow_type": "counter",
17676            "query_name": "current",
17677            "payload_codec": DEFAULT_CODEC,
17678            "workflow_arguments": empty_arguments.clone(),
17679            "query_arguments": empty_arguments
17680        }))
17681        .expect("query task");
17682        let unavailable = unavailable_worker
17683            .execute_query_task(unavailable_task)
17684            .await
17685            .expect_err("query handler unavailable");
17686        assert_eq!(unavailable.reason, "query_handler_unavailable");
17687    }
17688
17689    #[tokio::test]
17690    async fn client_query_decodes_result_and_typed_failure() {
17691        let server = MockWorkerServer::start();
17692        let client = Client::builder(server.base_url())
17693            .timeout(Duration::from_secs(2))
17694            .build()
17695            .expect("client");
17696
17697        let result = client
17698            .query_workflow("counter-1", "current", json!([]))
17699            .await
17700            .expect("query result");
17701        assert_eq!(result, json!({"count": 8}));
17702
17703        let error = client
17704            .query_workflow("counter-1", "missing", json!([]))
17705            .await
17706            .expect_err("unknown query");
17707        let Error::QueryFailed(failure) = error else {
17708            panic!("expected typed query failure");
17709        };
17710        assert_eq!(failure.status, 404);
17711        assert_eq!(failure.reason, "rejected_unknown_query");
17712    }
17713
17714    #[tokio::test]
17715    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
17716        let server = MockWorkerServer::start();
17717        let client = Client::builder(server.base_url())
17718            .timeout(Duration::from_secs(2))
17719            .build()
17720            .expect("client");
17721        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17722
17723        client
17724            .start_workflow(
17725                "typed.echo",
17726                "rust-workers",
17727                "typed-start",
17728                arguments.clone(),
17729            )
17730            .await
17731            .expect("typed workflow start");
17732        assert_eq!(
17733            decode_wire_avro_value(
17734                &server.request_body("/api/workflows")["input"],
17735                DEFAULT_CODEC,
17736            )
17737            .expect("typed start input"),
17738            arguments
17739        );
17740
17741        client
17742            .signal_workflow("typed-1", "changed", arguments.clone())
17743            .await
17744            .expect("typed signal");
17745        assert_eq!(
17746            decode_wire_avro_value(
17747                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
17748                DEFAULT_CODEC,
17749            )
17750            .expect("typed signal input"),
17751            arguments
17752        );
17753
17754        assert_eq!(
17755            client
17756                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
17757                .await
17758                .expect("typed query"),
17759            typed_fidelity_probe()
17760        );
17761        assert_eq!(
17762            decode_wire_avro_value(
17763                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
17764                DEFAULT_CODEC,
17765            )
17766            .expect("typed query input"),
17767            arguments
17768        );
17769
17770        assert_eq!(
17771            client
17772                .update_workflow_avro_value(
17773                    "typed-1",
17774                    "replace",
17775                    arguments.clone(),
17776                    Some("typed-request"),
17777                )
17778                .await
17779                .expect("typed update"),
17780            typed_fidelity_probe()
17781        );
17782        let update = server.request_body("/api/workflows/typed-1/update/replace");
17783        assert_eq!(update["request_id"], "typed-request");
17784        assert_eq!(
17785            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
17786            arguments
17787        );
17788
17789        let handle = WorkflowHandle {
17790            client: client.clone(),
17791            workflow_id: "typed-1".to_string(),
17792            run_id: Some("run-typed-1".to_string()),
17793            workflow_type: "typed.echo".to_string(),
17794        };
17795        assert_eq!(
17796            handle
17797                .result_avro_value(WorkflowResultOptions::default())
17798                .await
17799                .expect("typed workflow result"),
17800            typed_fidelity_probe()
17801        );
17802
17803        client
17804            .complete_activity_task(
17805                "activity-typed",
17806                "attempt-typed",
17807                "rust-worker",
17808                typed_fidelity_probe(),
17809                DEFAULT_CODEC,
17810            )
17811            .await
17812            .expect("typed activity completion");
17813        assert_eq!(
17814            decode_wire_avro_value(
17815                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
17816                    ["result"],
17817                DEFAULT_CODEC,
17818            )
17819            .expect("typed activity result"),
17820            typed_fidelity_probe()
17821        );
17822        client
17823            .fail_activity_task(
17824                "activity-typed",
17825                "attempt-typed",
17826                "rust-worker",
17827                "typed failure",
17828                true,
17829            )
17830            .await
17831            .expect("activity failure");
17832    }
17833
17834    #[tokio::test]
17835    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
17836        let server = MockWorkerServer::start();
17837        let client = Client::builder(server.base_url())
17838            .timeout(Duration::from_secs(2))
17839            .build()
17840            .expect("client");
17841
17842        let options = WorkflowCommandOptions::new()
17843            .reason("cleanup requested")
17844            .request_id("cancel-17");
17845        let cancelled = client
17846            .cancel_workflow("wf-lifecycle", options)
17847            .await
17848            .expect("instance cancellation");
17849        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
17850        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
17851        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
17852        assert_eq!(
17853            server.request_body("/api/workflows/wf-lifecycle/cancel"),
17854            json!({"reason":"cleanup requested","request_id":"cancel-17"})
17855        );
17856
17857        let terminated = client
17858            .terminate_workflow(
17859                "wf-lifecycle",
17860                WorkflowCommandOptions::new().reason("forced stop"),
17861            )
17862            .await
17863            .expect("instance termination");
17864        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
17865        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
17866
17867        client
17868            .cancel_workflow_run(
17869                "wf-lifecycle",
17870                "run-current",
17871                WorkflowCommandOptions::default(),
17872            )
17873            .await
17874            .expect("selected run cancellation");
17875        client
17876            .terminate_workflow_run(
17877                "wf-lifecycle",
17878                "run-current",
17879                WorkflowCommandOptions::default(),
17880            )
17881            .await
17882            .expect("selected run termination");
17883
17884        for (command, error) in [
17885            (
17886                WorkflowCommandKind::Cancel,
17887                client
17888                    .cancel_workflow_run(
17889                        "wf-lifecycle",
17890                        "run-stale",
17891                        WorkflowCommandOptions::default(),
17892                    )
17893                    .await
17894                    .expect_err("stale cancellation must be rejected"),
17895            ),
17896            (
17897                WorkflowCommandKind::Terminate,
17898                client
17899                    .terminate_workflow_run(
17900                        "wf-lifecycle",
17901                        "run-stale",
17902                        WorkflowCommandOptions::default(),
17903                    )
17904                    .await
17905                    .expect_err("stale termination must be rejected"),
17906            ),
17907        ] {
17908            let Error::WorkflowCommandRejected(rejection) = error else {
17909                panic!("expected typed command rejection");
17910            };
17911            assert_eq!(rejection.command, command);
17912            assert_eq!(rejection.status, 409);
17913            assert_eq!(rejection.reason, "historical_run_command_rejected");
17914            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
17915            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
17916        }
17917    }
17918
17919    #[tokio::test]
17920    async fn workflow_start_options_send_server_enforced_deadlines() {
17921        let server = MockWorkerServer::start();
17922        let client = Client::builder(server.base_url())
17923            .timeout(Duration::from_secs(2))
17924            .build()
17925            .expect("client");
17926
17927        let handle = client
17928            .start_workflow_with_options(
17929                "rust.timeout",
17930                "rust-timeouts",
17931                "wf-start-options",
17932                WorkflowStartOptions::new()
17933                    .execution_timeout_seconds(30)
17934                    .run_timeout_seconds(1),
17935                json!([]),
17936            )
17937            .await
17938            .expect("workflow start");
17939
17940        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
17941        let body = server.request_body("/api/workflows");
17942        assert_eq!(body["execution_timeout_seconds"], 30);
17943        assert_eq!(body["run_timeout_seconds"], 1);
17944
17945        let invalid = client
17946            .start_workflow_with_options(
17947                "rust.timeout",
17948                "rust-timeouts",
17949                "wf-invalid-options",
17950                WorkflowStartOptions::new()
17951                    .execution_timeout_seconds(1)
17952                    .run_timeout_seconds(2),
17953                json!([]),
17954            )
17955            .await
17956            .expect_err("invalid deadline ordering");
17957        assert!(invalid
17958            .to_string()
17959            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
17960    }
17961
17962    #[tokio::test]
17963    async fn workflow_result_returns_each_typed_terminal_outcome() {
17964        let server = MockWorkerServer::start();
17965        let client = Client::builder(server.base_url())
17966            .timeout(Duration::from_secs(2))
17967            .build()
17968            .expect("client");
17969        let options = WorkflowResultOptions {
17970            poll_interval: Duration::ZERO,
17971            timeout: Duration::from_secs(1),
17972        };
17973
17974        let failed = WorkflowHandle {
17975            client: client.clone(),
17976            workflow_id: "wf-failed".to_string(),
17977            run_id: Some("run-failed".to_string()),
17978            workflow_type: "failure".to_string(),
17979        }
17980        .result(options)
17981        .await
17982        .expect_err("failed outcome");
17983        let Error::WorkflowFailed(failure) = failed else {
17984            panic!("expected WorkflowFailed");
17985        };
17986        assert_eq!(failure.workflow_id, "wf-failed");
17987        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
17988        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
17989        assert_eq!(failure.failure_category.as_deref(), Some("application"));
17990        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
17991        assert_eq!(
17992            failure.exception_class.as_deref(),
17993            Some("billing::PaymentError")
17994        );
17995        assert_eq!(failure.non_retryable, Some(true));
17996
17997        for (workflow_id, expected_kind, expected_reason) in [
17998            (
17999                "wf-cancelled",
18000                WorkflowTerminalKind::Cancelled,
18001                "cleanup requested",
18002            ),
18003            (
18004                "wf-terminated",
18005                WorkflowTerminalKind::Terminated,
18006                "forced stop",
18007            ),
18008            (
18009                "wf-timed-out",
18010                WorkflowTerminalKind::TimedOut,
18011                "run_timeout",
18012            ),
18013        ] {
18014            let error = WorkflowHandle {
18015                client: client.clone(),
18016                workflow_id: workflow_id.to_string(),
18017                run_id: None,
18018                workflow_type: "terminal".to_string(),
18019            }
18020            .result(options)
18021            .await
18022            .expect_err("typed terminal outcome");
18023            let outcome = match error {
18024                Error::WorkflowCancelled(outcome) => outcome,
18025                Error::WorkflowTerminated(outcome) => outcome,
18026                Error::WorkflowTimedOut(outcome) => outcome,
18027                other => panic!("unexpected terminal error: {other}"),
18028            };
18029            assert_eq!(outcome.kind, expected_kind);
18030            assert_eq!(outcome.workflow_id, workflow_id);
18031            assert_eq!(outcome.reason, expected_reason);
18032        }
18033
18034        let wait_timeout = WorkflowHandle {
18035            client,
18036            workflow_id: "wf-waiting".to_string(),
18037            run_id: Some("run-waiting".to_string()),
18038            workflow_type: "waiting".to_string(),
18039        }
18040        .result(WorkflowResultOptions {
18041            poll_interval: Duration::ZERO,
18042            timeout: Duration::ZERO,
18043        })
18044        .await
18045        .expect_err("client wait timeout");
18046        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
18047            panic!("expected typed client timeout");
18048        };
18049        assert_eq!(timeout.reason, "result_wait_timeout");
18050        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
18051        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
18052    }
18053
18054    #[tokio::test]
18055    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
18056        let server = MockWorkerServer::start();
18057        let client = Client::builder(server.base_url())
18058            .timeout(Duration::from_secs(2))
18059            .build()
18060            .expect("client");
18061
18062        let handle = WorkflowHandle {
18063            client,
18064            workflow_id: "wf-selected".to_string(),
18065            run_id: Some("run-selected".to_string()),
18066            workflow_type: "selected".to_string(),
18067        };
18068        let options = WorkflowResultOptions {
18069            poll_interval: Duration::ZERO,
18070            timeout: Duration::from_secs(1),
18071        };
18072
18073        let current = handle
18074            .result(options)
18075            .await
18076            .expect("instance result follows the current run");
18077        assert_eq!(current, json!("current run output"));
18078
18079        let error = handle
18080            .result_selected_run(options)
18081            .await
18082            .expect_err("the selected run is cancelled even though the current run completed");
18083
18084        let Error::WorkflowCancelled(outcome) = error else {
18085            panic!("expected selected run cancellation");
18086        };
18087        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
18088        assert_eq!(outcome.reason, "selected run cancelled");
18089        assert_eq!(
18090            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
18091            1
18092        );
18093        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
18094    }
18095
18096    #[tokio::test]
18097    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
18098        let server = MockWorkerServer::draining_polls();
18099        let client = Client::builder(server.base_url())
18100            .timeout(Duration::from_secs(2))
18101            .build()
18102            .expect("client");
18103
18104        let workflow = client
18105            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
18106            .await
18107            .expect("workflow drain response");
18108        let activity = client
18109            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
18110            .await
18111            .expect("activity drain response");
18112        let query = client
18113            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
18114            .await
18115            .expect("query drain response");
18116
18117        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
18118            assert_eq!(
18119                outcome,
18120                WorkerPollOutcome::Stop {
18121                    poll_status: Some("draining".to_string()),
18122                    reason: Some("worker_draining".to_string()),
18123                }
18124            );
18125        }
18126
18127        assert!(client
18128            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
18129            .await
18130            .expect("compatibility poll")
18131            .is_none());
18132    }
18133
18134    #[tokio::test]
18135    async fn managed_worker_honors_drain_stop_for_every_task_family() {
18136        let server = MockWorkerServer::draining_polls();
18137        let client = Client::builder(server.base_url())
18138            .timeout(Duration::from_secs(2))
18139            .build()
18140            .expect("client");
18141
18142        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
18143            .worker_id("draining-workflow-worker")
18144            .poll_timeout(Duration::ZERO);
18145        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
18146        workflow_worker
18147            .run()
18148            .await
18149            .expect("workflow drain is a clean stop");
18150
18151        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
18152            .worker_id("draining-activity-worker")
18153            .poll_timeout(Duration::ZERO);
18154        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
18155        activity_worker
18156            .run()
18157            .await
18158            .expect("activity drain is a clean stop");
18159
18160        let mut query_worker = Worker::new(client, "rust-workers")
18161            .worker_id("draining-query-worker")
18162            .poll_timeout(Duration::ZERO);
18163        query_worker.register_query("counter", "current", |_ctx, _args| async {
18164            Ok(Value::Null)
18165        });
18166        query_worker
18167            .run()
18168            .await
18169            .expect("query drain is a clean stop");
18170    }
18171
18172    #[tokio::test]
18173    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
18174        let server = MockWorkerServer::start();
18175        let client = Client::builder(server.base_url())
18176            .timeout(Duration::from_secs(2))
18177            .build()
18178            .expect("client");
18179
18180        let heartbeat = client
18181            .heartbeat_activity_task(
18182                "activity-cancel",
18183                "attempt-cancel",
18184                "rust-worker",
18185                typed_fidelity_probe(),
18186            )
18187            .await
18188            .expect("cancellation heartbeat");
18189        assert!(heartbeat.cancel_requested);
18190        assert!(heartbeat.should_stop());
18191        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
18192        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
18193        let heartbeat_body =
18194            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
18195        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
18196        assert_eq!(
18197            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
18198                .expect("typed heartbeat details"),
18199            typed_fidelity_probe()
18200        );
18201
18202        let error = client
18203            .complete_activity_task(
18204                "activity-cancel",
18205                "attempt-cancel",
18206                "rust-worker",
18207                json!({"late":true}),
18208                DEFAULT_CODEC,
18209            )
18210            .await
18211            .expect_err("late completion must be refused");
18212        assert!(activity_task_rejection_is_final(&error));
18213        let Error::ActivityTaskRejected(rejection) = error else {
18214            panic!("expected typed activity rejection");
18215        };
18216        assert_eq!(rejection.status, 409);
18217        assert_eq!(rejection.reason, "run_cancelled");
18218        assert!(rejection.cancel_requested);
18219        assert_eq!(rejection.can_continue, Some(false));
18220    }
18221
18222    #[tokio::test]
18223    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
18224        let server = MockWorkerServer::cancelled_activity();
18225        let client = Client::builder(server.base_url())
18226            .timeout(Duration::from_secs(2))
18227            .build()
18228            .expect("client");
18229        let cancellation_observed = Arc::new(AtomicBool::new(false));
18230        let observed = Arc::clone(&cancellation_observed);
18231        let mut worker = Worker::new(client.clone(), "rust-workers")
18232            .worker_id("rust-cancel-worker")
18233            .poll_timeout(Duration::from_millis(10));
18234        worker.register_activity("cancel-aware", move |ctx, _args| {
18235            let observed = Arc::clone(&observed);
18236            async move {
18237                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
18238                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
18239                Ok(json!({"late":"completion"}))
18240            }
18241        });
18242
18243        assert_eq!(
18244            worker.run_once().await.expect("cancelled attempt handled"),
18245            1
18246        );
18247        assert!(cancellation_observed.load(Ordering::SeqCst));
18248        assert_eq!(
18249            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
18250            1
18251        );
18252
18253        let mut restarted = Worker::new(client, "rust-workers")
18254            .worker_id("rust-cancel-worker-restarted")
18255            .poll_timeout(Duration::from_millis(10));
18256        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
18257        assert_eq!(
18258            restarted
18259                .run_once()
18260                .await
18261                .expect("replacement worker continues polling"),
18262            0
18263        );
18264    }
18265
18266    #[tokio::test]
18267    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
18268        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
18269        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
18270        let client = Client::builder(server.base_url())
18271            .timeout(Duration::from_secs(2))
18272            .build()
18273            .expect("client");
18274
18275        let direct_error = client
18276            .complete_workflow_task(
18277                "workflow-timeout-task",
18278                "timeout-worker",
18279                3,
18280                vec![json!({
18281                    "type": "complete_workflow",
18282                    "result": fixture_envelope(Value::Null)
18283                })],
18284            )
18285            .await
18286            .expect_err("the low-level client preserves the completion rejection");
18287        let Error::Http { status, body } = direct_error else {
18288            panic!("expected the original HTTP completion rejection");
18289        };
18290        assert_eq!(status, reqwest::StatusCode::CONFLICT);
18291        assert_eq!(
18292            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
18293            "run_timed_out"
18294        );
18295
18296        let mut worker = Worker::new(client, "rust-workers")
18297            .worker_id("timeout-worker")
18298            .poll_timeout(Duration::from_millis(10));
18299        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
18300            Ok(json!({"late": "result"}))
18301        });
18302
18303        assert_eq!(
18304            worker
18305                .run_once()
18306                .await
18307                .expect("authoritative selected-run timeout settles the tick"),
18308            1
18309        );
18310        assert_eq!(
18311            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
18312            2,
18313            "both the direct client proof and managed worker must see the rejection"
18314        );
18315    }
18316
18317    #[tokio::test]
18318    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
18319        for (name, status, response) in [
18320            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
18321            (
18322                "command was recorded",
18323                "409 Conflict",
18324                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18325            ),
18326            (
18327                "lease conflict",
18328                "409 Conflict",
18329                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
18330            ),
18331            (
18332                "nonterminal run",
18333                "409 Conflict",
18334                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
18335            ),
18336            (
18337                "different selected run",
18338                "409 Conflict",
18339                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
18340            ),
18341            (
18342                "different task attempt",
18343                "409 Conflict",
18344                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18345            ),
18346            (
18347                "authentication failure",
18348                "401 Unauthorized",
18349                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18350            ),
18351            (
18352                "authorization failure",
18353                "403 Forbidden",
18354                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18355            ),
18356            (
18357                "protocol failure",
18358                "400 Bad Request",
18359                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
18360            ),
18361            (
18362                "malformed command",
18363                "422 Unprocessable Entity",
18364                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18365            ),
18366            (
18367                "transient server failure",
18368                "503 Service Unavailable",
18369                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
18370            ),
18371        ] {
18372            let server = MockWorkerServer::workflow_completion(status, response);
18373            let client = Client::builder(server.base_url())
18374                .timeout(Duration::from_secs(2))
18375                .build()
18376                .expect("client");
18377            let mut worker = Worker::new(client, "rust-workers")
18378                .worker_id("timeout-worker")
18379                .poll_timeout(Duration::from_millis(10));
18380            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
18381                Ok(json!({"late": "result"}))
18382            });
18383
18384            let error = worker
18385                .run_once()
18386                .await
18387                .expect_err(&format!("{name} must remain an error"));
18388            assert!(
18389                matches!(error, Error::Http { .. } | Error::Protocol(_)),
18390                "{name} returned an unexpected error variant: {error}"
18391            );
18392        }
18393    }
18394
18395    #[tokio::test]
18396    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
18397        let server = MockWorkerServer::start();
18398        let client = Client::builder(server.base_url())
18399            .worker_token(Some("worker-secret".to_string()))
18400            .namespace("orders")
18401            .timeout(Duration::from_secs(2))
18402            .build()
18403            .expect("client");
18404        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
18405
18406        let result = client
18407            .deregister_worker_registration("worker/α space")
18408            .await
18409            .expect("deregister worker registration");
18410
18411        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
18412        assert_eq!(
18413            server.worker_protocol_for(path).as_deref(),
18414            Some(WORKER_PROTOCOL_VERSION)
18415        );
18416        assert_eq!(server.control_protocol_for(path), None);
18417        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
18418        assert_eq!(
18419            server.authorization_for(path).as_deref(),
18420            Some("Bearer worker-secret")
18421        );
18422        assert_eq!(
18423            result,
18424            WorkerDeregistrationEnvelope {
18425                worker_id: "deregistered-worker".to_string(),
18426                outcome: "deregistered".to_string(),
18427                recovered_workflow_task_count: 2,
18428            }
18429        );
18430    }
18431
18432    #[tokio::test]
18433    async fn low_level_registration_rejects_update_validators_before_transport() {
18434        let server = MockWorkerServer::start();
18435        let client = Client::builder(server.base_url())
18436            .timeout(Duration::from_secs(2))
18437            .build()
18438            .expect("client");
18439
18440        for update_validators in [json!(["approve"]), json!("approve")] {
18441            let error = client
18442                .register_worker_with_command_contracts(
18443                    "validator-claiming-worker",
18444                    "rust-workers",
18445                    vec!["orders".to_string()],
18446                    vec![],
18447                    1,
18448                    1,
18449                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
18450                    json!({
18451                        "orders": {
18452                            "queries": ["current"],
18453                            "updates": ["approve"],
18454                            "update_validators": update_validators,
18455                        },
18456                    }),
18457                )
18458                .await
18459                .expect_err("unsupported validator claims must fail before registration");
18460
18461            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
18462                panic!("expected typed unsupported-validator failure");
18463            };
18464            assert_eq!(workflow_type, "orders");
18465        }
18466        assert_eq!(server.request_count("/api/worker/register"), 0);
18467    }
18468
18469    #[tokio::test]
18470    async fn low_level_registration_preserves_query_and_update_contracts() {
18471        let server = MockWorkerServer::start();
18472        let client = Client::builder(server.base_url())
18473            .timeout(Duration::from_secs(2))
18474            .build()
18475            .expect("client");
18476        let contracts = json!({
18477            "orders": {
18478                "queries": ["current"],
18479                "updates": ["approve"],
18480                "update_validators": [],
18481            },
18482            "payments": {
18483                "queries": ["status"],
18484                "updates": ["capture"],
18485            },
18486        });
18487
18488        client
18489            .register_worker_with_command_contracts(
18490                "command-worker",
18491                "rust-workers",
18492                vec!["orders".to_string(), "payments".to_string()],
18493                vec![],
18494                1,
18495                1,
18496                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
18497                contracts.clone(),
18498            )
18499            .await
18500            .expect("query and update contracts must remain supported");
18501
18502        assert_eq!(
18503            server.request_body("/api/worker/register")["workflow_command_contracts"],
18504            contracts
18505        );
18506    }
18507
18508    #[tokio::test]
18509    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
18510        let server = MockWorkerServer::start();
18511        let control_only = Client::builder(server.base_url())
18512            .control_token(Some("control-secret".to_string()))
18513            .build()
18514            .expect("control client");
18515
18516        let error = control_only
18517            .register_worker("worker", "queue", vec![], vec![], 1, 1)
18518            .await
18519            .expect_err("control token must not authorize a worker request");
18520        assert!(matches!(
18521            error,
18522            Error::MissingRoleCredentials { role: "worker", .. }
18523        ));
18524        assert_eq!(server.request_count("/api/worker/register"), 0);
18525
18526        let worker_only = Client::builder(server.base_url())
18527            .worker_token(Some("worker-secret".to_string()))
18528            .build()
18529            .expect("worker client");
18530        let error = worker_only
18531            .health()
18532            .await
18533            .expect_err("worker token must not authorize a control request");
18534        assert!(matches!(
18535            error,
18536            Error::MissingRoleCredentials {
18537                role: "control",
18538                ..
18539            }
18540        ));
18541        assert_eq!(server.request_count("/api/health"), 0);
18542    }
18543
18544    #[tokio::test]
18545    async fn shared_token_supports_worker_and_control_planes() {
18546        let server = MockWorkerServer::start();
18547        let client = Client::builder(server.base_url())
18548            .token(Some("shared-secret".to_string()))
18549            .build()
18550            .expect("client");
18551
18552        client.health().await.expect("control request");
18553        client
18554            .register_worker("worker", "queue", vec![], vec![], 1, 1)
18555            .await
18556            .expect("worker request");
18557
18558        assert_eq!(
18559            server.authorization_for("/api/health").as_deref(),
18560            Some("Bearer shared-secret")
18561        );
18562        assert_eq!(
18563            server.control_protocol_for("/api/health").as_deref(),
18564            Some(CONTROL_PLANE_VERSION)
18565        );
18566        assert_eq!(
18567            server.authorization_for("/api/worker/register").as_deref(),
18568            Some("Bearer shared-secret")
18569        );
18570        assert_eq!(
18571            server
18572                .worker_protocol_for("/api/worker/register")
18573                .as_deref(),
18574            Some(WORKER_PROTOCOL_VERSION)
18575        );
18576    }
18577
18578    #[tokio::test]
18579    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
18580        let server = MockWorkerServer::start();
18581        let client = Client::builder(server.base_url())
18582            .timeout(Duration::from_secs(2))
18583            .build()
18584            .expect("client");
18585
18586        client
18587            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
18588            .await
18589            .expect("register");
18590        client
18591            .heartbeat_worker("capture-worker", 1, 1)
18592            .await
18593            .expect("heartbeat");
18594        client
18595            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
18596            .await
18597            .expect("workflow poll");
18598        client
18599            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
18600            .await
18601            .expect("activity poll");
18602
18603        for path in [
18604            "/api/worker/register",
18605            "/api/worker/heartbeat",
18606            "/api/worker/workflow-tasks/poll",
18607            "/api/worker/activity-tasks/poll",
18608        ] {
18609            assert_eq!(
18610                server.worker_protocol_for(path).as_deref(),
18611                Some(WORKER_PROTOCOL_VERSION),
18612                "unexpected protocol for {path}"
18613            );
18614        }
18615
18616        assert_eq!(
18617            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
18618            1
18619        );
18620        assert_eq!(
18621            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
18622            1
18623        );
18624        assert!(
18625            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
18626                .as_str()
18627                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
18628        );
18629        assert!(
18630            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
18631                .as_str()
18632                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
18633        );
18634    }
18635
18636    #[tokio::test]
18637    async fn query_task_endpoints_send_the_query_feature_protocol() {
18638        let server = MockWorkerServer::start();
18639        let client = Client::builder(server.base_url())
18640            .timeout(Duration::from_secs(2))
18641            .build()
18642            .expect("client");
18643
18644        client
18645            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
18646            .await
18647            .expect("query poll");
18648        client
18649            .complete_query_task(
18650                "query-capture",
18651                "capture-worker",
18652                1,
18653                json!(8),
18654                DEFAULT_CODEC,
18655            )
18656            .await
18657            .expect("query complete");
18658        client
18659            .fail_query_task(
18660                "query-capture",
18661                "capture-worker",
18662                1,
18663                "failed",
18664                "query_rejected",
18665                "QueryFailed",
18666            )
18667            .await
18668            .expect("query fail");
18669
18670        for path in [
18671            "/api/worker/query-tasks/poll",
18672            "/api/worker/query-tasks/query-capture/complete",
18673            "/api/worker/query-tasks/query-capture/fail",
18674        ] {
18675            assert_eq!(
18676                server.worker_protocol_for(path).as_deref(),
18677                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
18678                "unexpected protocol for {path}"
18679            );
18680        }
18681
18682        assert_eq!(
18683            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
18684            1
18685        );
18686        assert!(
18687            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
18688                .as_str()
18689                .is_some_and(|id| id.starts_with("rust-query-poll-"))
18690        );
18691    }
18692
18693    #[tokio::test]
18694    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
18695        let server = MockWorkerServer::transient_worker_failures();
18696        let client = Client::builder(server.base_url())
18697            .timeout(Duration::from_secs(2))
18698            .build()
18699            .expect("client");
18700
18701        client
18702            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
18703            .await
18704            .expect("workflow poll retry");
18705        client
18706            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
18707            .await
18708            .expect("activity poll retry");
18709        client
18710            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
18711            .await
18712            .expect("query poll retry");
18713
18714        for path in [
18715            "/api/worker/workflow-tasks/poll",
18716            "/api/worker/activity-tasks/poll",
18717            "/api/worker/query-tasks/poll",
18718        ] {
18719            let bodies = server.request_bodies(path);
18720            assert_eq!(bodies.len(), 2, "{path} must be retried once");
18721            assert_eq!(
18722                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
18723                "{path} must preserve the request binding across retry"
18724            );
18725        }
18726    }
18727
18728    #[tokio::test]
18729    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
18730        let server = MockWorkerServer::consecutive_poll_failures(2);
18731        let client = Client::builder(server.base_url())
18732            .timeout(Duration::from_secs(2))
18733            .build()
18734            .expect("client");
18735        let mut worker = Worker::new(client, "capture")
18736            .worker_id("capture-worker")
18737            .poll_timeout(Duration::from_millis(10))
18738            .retry_policy(WorkerRetryPolicy {
18739                max_retries: 2,
18740                initial_backoff: Duration::from_millis(1),
18741                max_backoff: Duration::from_millis(1),
18742            });
18743        worker.register_workflow(
18744            "capture.workflow",
18745            |_ctx, _input| async move { Ok(Value::Null) },
18746        );
18747        worker.register_activity(
18748            "capture.activity",
18749            |_ctx, _input| async move { Ok(Value::Null) },
18750        );
18751        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
18752            Ok(Value::Null)
18753        });
18754
18755        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
18756
18757        for path in [
18758            "/api/worker/workflow-tasks/poll",
18759            "/api/worker/activity-tasks/poll",
18760            "/api/worker/query-tasks/poll",
18761        ] {
18762            let bodies = server.request_bodies(path);
18763            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
18764            assert!(
18765                bodies
18766                    .iter()
18767                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
18768                "{path} must preserve one request binding across every retry"
18769            );
18770        }
18771    }
18772
18773    #[tokio::test]
18774    async fn query_protocol_rejection_from_older_server_is_typed() {
18775        let server = MockWorkerServer::reject_query_protocol();
18776        let client = Client::builder(server.base_url())
18777            .timeout(Duration::from_secs(2))
18778            .build()
18779            .expect("client");
18780
18781        let error = client
18782            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
18783            .await
18784            .expect_err("server below query protocol floor must reject");
18785        let Error::Protocol(failure) = error else {
18786            panic!("expected typed protocol failure");
18787        };
18788
18789        assert_eq!(failure.status, 400);
18790        assert_eq!(failure.reason, "unsupported_protocol_version");
18791        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
18792        assert_eq!(
18793            failure.requested_version.as_deref(),
18794            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
18795        );
18796        assert_eq!(
18797            server
18798                .worker_protocol_for("/api/worker/query-tasks/poll")
18799                .as_deref(),
18800            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
18801        );
18802    }
18803
18804    #[tokio::test]
18805    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
18806        let server = MockWorkerServer::reject_query_protocol();
18807        let client = Client::builder(server.base_url())
18808            .timeout(Duration::from_secs(2))
18809            .build()
18810            .expect("client");
18811        let mut worker = Worker::new(client, "rust-workers")
18812            .worker_id("baseline-worker")
18813            .poll_timeout(Duration::from_millis(10));
18814
18815        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
18816            Ok(Value::Null)
18817        });
18818
18819        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
18820        assert_eq!(
18821            server
18822                .worker_protocol_for("/api/worker/workflow-tasks/poll")
18823                .as_deref(),
18824            Some(WORKER_PROTOCOL_VERSION)
18825        );
18826        assert_eq!(
18827            server.worker_protocol_for("/api/worker/query-tasks/poll"),
18828            None,
18829            "a worker without query handlers must not use the query-task endpoint"
18830        );
18831    }
18832
18833    #[tokio::test]
18834    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
18835        let server = MockWorkerServer::reject_query_completion();
18836        let client = Client::builder(server.base_url())
18837            .timeout(Duration::from_secs(2))
18838            .build()
18839            .expect("client");
18840
18841        let error = client
18842            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
18843            .await
18844            .expect_err("expired completion must be rejected");
18845        let Error::QueryFailed(failure) = error else {
18846            panic!("expected typed query failure");
18847        };
18848        assert_eq!(failure.status, 409);
18849        assert_eq!(failure.reason, "query_task_timed_out");
18850
18851        let mut worker = Worker::new(client, "rust-workers")
18852            .worker_id("late-worker")
18853            .poll_timeout(Duration::from_millis(10));
18854        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
18855        worker.register_query(
18856            "counter",
18857            "current",
18858            |_ctx, _args| async move { Ok(json!(8)) },
18859        );
18860
18861        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
18862        assert_eq!(
18863            worker
18864                .run_once()
18865                .await
18866                .expect("worker continues after late completion"),
18867            0
18868        );
18869        assert_eq!(
18870            server.request_count("/api/worker/query-tasks/query-late/complete"),
18871            2
18872        );
18873        assert_eq!(
18874            server.request_count("/api/worker/query-tasks/query-late/fail"),
18875            0,
18876            "a server completion rejection must not be reported as an encoding failure"
18877        );
18878    }
18879
18880    #[tokio::test]
18881    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
18882        let server = MockWorkerServer::start();
18883        let client = Client::builder(server.base_url())
18884            .timeout(Duration::from_secs(2))
18885            .build()
18886            .expect("client");
18887        let mut worker = Worker::new(client, "rust-workers")
18888            .worker_id("joined-worker")
18889            .poll_timeout(Duration::from_millis(10));
18890        worker.register_workflow(
18891            "joined.workflow",
18892            |_ctx, _input| async move { Ok(Value::Null) },
18893        );
18894        worker.register_activity(
18895            "joined.activity",
18896            |_ctx, _input| async move { Ok(Value::Null) },
18897        );
18898        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
18899            Ok(Value::Null)
18900        });
18901
18902        worker
18903            .run_until(tokio::time::sleep(Duration::from_millis(20)))
18904            .await
18905            .expect("normal shutdown");
18906
18907        let deregistration_path = "/api/worker/registrations/mock-worker";
18908        assert_eq!(server.request_count(deregistration_path), 1);
18909        for poll_path in [
18910            "/api/worker/workflow-tasks/poll",
18911            "/api/worker/activity-tasks/poll",
18912            "/api/worker/query-tasks/poll",
18913        ] {
18914            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
18915        }
18916        assert_eq!(
18917            server.captured_paths().last().map(String::as_str),
18918            Some(deregistration_path),
18919            "deregistration must start only after every poller has joined"
18920        );
18921    }
18922
18923    #[tokio::test]
18924    async fn registration_failure_does_not_deregister() {
18925        let server = MockWorkerServer::rejected_registration();
18926        let client = Client::builder(server.base_url())
18927            .timeout(Duration::from_secs(2))
18928            .build()
18929            .expect("client");
18930        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
18931
18932        let error = worker
18933            .run_until(async {})
18934            .await
18935            .expect_err("registration must fail");
18936        assert!(matches!(
18937            error,
18938            Error::Http {
18939                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
18940                ..
18941            }
18942        ));
18943        assert!(server
18944            .captured_paths()
18945            .iter()
18946            .all(|path| !path.starts_with("/api/worker/registrations/")));
18947    }
18948
18949    #[tokio::test]
18950    async fn declined_registration_does_not_deregister() {
18951        let server = MockWorkerServer::declined_registration();
18952        let client = Client::builder(server.base_url())
18953            .timeout(Duration::from_secs(2))
18954            .build()
18955            .expect("client");
18956        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
18957
18958        let error = worker
18959            .run_until(async {})
18960            .await
18961            .expect_err("declined registration must fail");
18962        assert!(matches!(error, Error::WorkerLoop(_)));
18963        assert!(error.to_string().contains("was not accepted"));
18964        assert!(server
18965            .captured_paths()
18966            .iter()
18967            .all(|path| !path.starts_with("/api/worker/registrations/")));
18968    }
18969
18970    #[tokio::test]
18971    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
18972        let server = MockWorkerServer::rejected_deregistration();
18973        let client = Client::builder(server.base_url())
18974            .timeout(Duration::from_secs(2))
18975            .build()
18976            .expect("client");
18977        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
18978
18979        let error = worker
18980            .run_until(async {})
18981            .await
18982            .expect_err("deregistration must fail");
18983        assert!(matches!(
18984            error,
18985            Error::Http {
18986                status: reqwest::StatusCode::FORBIDDEN,
18987                ..
18988            }
18989        ));
18990        assert_eq!(
18991            server.request_count("/api/worker/registrations/mock-worker"),
18992            1
18993        );
18994    }
18995
18996    #[tokio::test]
18997    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
18998        let server = MockWorkerServer::rejected_deregistration_protocol();
18999        let client = Client::builder(server.base_url())
19000            .timeout(Duration::from_secs(2))
19001            .build()
19002            .expect("client");
19003        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
19004
19005        let error = worker
19006            .run_until(async {})
19007            .await
19008            .expect_err("protocol rejection must fail shutdown");
19009        let Error::Protocol(failure) = error else {
19010            panic!("expected typed protocol failure");
19011        };
19012        assert_eq!(failure.reason, "unsupported_protocol_version");
19013        assert_eq!(
19014            failure.requested_version.as_deref(),
19015            Some(WORKER_PROTOCOL_VERSION)
19016        );
19017        assert_eq!(
19018            server.request_count("/api/worker/registrations/mock-worker"),
19019            1
19020        );
19021    }
19022
19023    #[tokio::test]
19024    async fn primary_poller_error_retains_deregistration_failure_context() {
19025        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
19026        let client = Client::builder(server.base_url())
19027            .timeout(Duration::from_secs(2))
19028            .build()
19029            .expect("client");
19030        let mut worker = Worker::new(client, "rust-workers")
19031            .worker_id("combined-failure")
19032            .poll_timeout(Duration::from_millis(10));
19033        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
19034            Ok(Value::Null)
19035        });
19036
19037        let error = worker
19038            .run()
19039            .await
19040            .expect_err("worker and cleanup must fail");
19041        let summary = error.to_string();
19042        assert!(summary.contains("authentication_failed"));
19043        assert!(summary.contains("worker cannot deregister"));
19044        let Error::WorkerShutdown {
19045            primary,
19046            deregistration,
19047        } = error
19048        else {
19049            panic!("expected combined worker shutdown error");
19050        };
19051        assert!(matches!(
19052            *primary,
19053            Error::Http {
19054                status: reqwest::StatusCode::UNAUTHORIZED,
19055                ..
19056            }
19057        ));
19058        assert!(matches!(
19059            *deregistration,
19060            Error::Http {
19061                status: reqwest::StatusCode::FORBIDDEN,
19062                ..
19063            }
19064        ));
19065        assert_eq!(
19066            server.request_count("/api/worker/registrations/mock-worker"),
19067            1
19068        );
19069    }
19070
19071    #[tokio::test]
19072    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
19073        let server = MockWorkerServer::start();
19074        let client = Client::builder(server.base_url())
19075            .timeout(Duration::from_secs(2))
19076            .build()
19077            .expect("client");
19078        let mut worker = Worker::new(client, "rust-workers")
19079            .worker_id("activity-only-worker")
19080            .poll_timeout(Duration::from_millis(10));
19081
19082        worker.register_activity(
19083            "activity.only",
19084            |_ctx, _args| async move { Ok(Value::Null) },
19085        );
19086
19087        worker.run_until(async {}).await.expect("run worker");
19088    }
19089
19090    #[tokio::test]
19091    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
19092        let server = MockWorkerServer::start();
19093        let client = Client::builder(server.base_url())
19094            .timeout(Duration::from_secs(2))
19095            .build()
19096            .expect("client");
19097        let mut worker = Worker::new(client, "rust-workers")
19098            .worker_id("workflow-only-worker")
19099            .poll_timeout(Duration::from_millis(10));
19100
19101        worker.register_workflow(
19102            "workflow.only",
19103            |_ctx, _input| async move { Ok(Value::Null) },
19104        );
19105
19106        worker.run_until(async {}).await.expect("run worker");
19107    }
19108
19109    #[tokio::test]
19110    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
19111        let server = MockWorkerServer::start();
19112        let client = Client::builder(server.base_url())
19113            .timeout(Duration::from_secs(2))
19114            .build()
19115            .expect("client");
19116        let observations = Arc::new(Mutex::new(Vec::new()));
19117        let observed = Arc::clone(&observations);
19118        let mut worker = Worker::new(client, "rust-workers")
19119            .worker_id("observed-heartbeat-worker")
19120            .poll_timeout(Duration::from_millis(10))
19121            .on_worker_heartbeat(move |observation| {
19122                observed
19123                    .lock()
19124                    .expect("heartbeat observations")
19125                    .push(observation.clone());
19126            });
19127
19128        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
19129            Ok(Value::Null)
19130        });
19131        let acknowledged = Arc::clone(&observations);
19132        worker
19133            .run_until(async move {
19134                tokio::time::timeout(Duration::from_secs(2), async move {
19135                    loop {
19136                        if !acknowledged
19137                            .lock()
19138                            .expect("heartbeat observations")
19139                            .is_empty()
19140                        {
19141                            break;
19142                        }
19143                        tokio::time::sleep(Duration::from_millis(1)).await;
19144                    }
19145                })
19146                .await
19147                .expect("heartbeat acknowledgement within timeout");
19148            })
19149            .await
19150            .expect("run worker");
19151
19152        let observations = observations.lock().expect("heartbeat observations");
19153        let first = observations.first().expect("heartbeat acknowledgement");
19154        assert_eq!(first.worker_id, "observed-heartbeat-worker");
19155        assert_eq!(first.task_queue, "rust-workers");
19156        assert!(first.acknowledged_at_unix_millis > 0);
19157        assert_eq!(first.acknowledgement, json!({}));
19158    }
19159
19160    #[tokio::test]
19161    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
19162        let server = MockWorkerServer::delayed_heartbeat_worker();
19163        let client = Client::builder(server.base_url())
19164            .timeout(Duration::from_secs(3))
19165            .build()
19166            .expect("client");
19167        let observations = Arc::new(Mutex::new(Vec::new()));
19168        let observed = Arc::clone(&observations);
19169        let mut worker = Worker::new(client, "rust-snapshot-workers")
19170            .worker_id("rust-snapshot-worker")
19171            .poll_timeout(Duration::from_millis(10))
19172            .on_worker_heartbeat(move |observation| {
19173                observed
19174                    .lock()
19175                    .expect("heartbeat observations")
19176                    .push(observation.clone());
19177            });
19178
19179        worker.register_workflow("snapshot", |ctx, _input| async move {
19180            ctx.wait_signal("finish").await?;
19181            Ok(json!({"status": "finished"}))
19182        });
19183        worker.register_query("snapshot", "current", |ctx, _args| async move {
19184            Ok(json!(ctx
19185                .signals("increment")
19186                .iter()
19187                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
19188                .sum::<i64>()))
19189        });
19190        worker.register_activity("cancel-aware", |_ctx, _args| async move {
19191            Ok(json!({"late": "completion"}))
19192        });
19193
19194        worker
19195            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
19196            .await
19197            .expect("delayed heartbeat must allow a clean worker shutdown");
19198
19199        let observations = observations.lock().expect("heartbeat observations");
19200        assert!(
19201            observations.len() >= 3,
19202            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
19203        );
19204        assert!(
19205            observations.windows(2).all(|pair| {
19206                pair[1].acknowledged_at_unix_millis
19207                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
19208                    >= 850
19209            }),
19210            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
19211        );
19212        drop(observations);
19213
19214        let heartbeat_times = server.request_times("/api/worker/heartbeat");
19215        let delayed_request_at = *heartbeat_times
19216            .get(1)
19217            .expect("intentionally delayed heartbeat request");
19218        let delay_window_start = delayed_request_at + Duration::from_millis(100);
19219        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
19220        for path in [
19221            "/api/worker/workflow-tasks/poll",
19222            "/api/worker/activity-tasks/poll",
19223            "/api/worker/query-tasks/poll",
19224        ] {
19225            assert!(
19226                server
19227                    .request_times(path)
19228                    .iter()
19229                    .any(|received_at| *received_at >= delay_window_start
19230                        && *received_at <= delay_window_end),
19231                "{path} must keep polling while a heartbeat acknowledgement is delayed"
19232            );
19233        }
19234        assert!(
19235            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
19236            "workflow work must be settled"
19237        );
19238        assert!(
19239            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
19240            "activity work must be settled"
19241        );
19242        assert!(
19243            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
19244            "query work must be settled"
19245        );
19246    }
19247
19248    #[tokio::test]
19249    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
19250        let server = MockWorkerServer::heartbeat_retry_worker();
19251        let client = Client::builder(server.base_url())
19252            .timeout(Duration::from_secs(2))
19253            .build()
19254            .expect("client");
19255        let observations = Arc::new(Mutex::new(Vec::new()));
19256        let observed = Arc::clone(&observations);
19257        let worker = Worker::new(client, "rust-workers")
19258            .worker_id("heartbeat-retry-worker")
19259            .retry_policy(WorkerRetryPolicy {
19260                max_retries: 1,
19261                initial_backoff: Duration::from_millis(300),
19262                max_backoff: Duration::from_millis(300),
19263            })
19264            .on_worker_heartbeat(move |observation| {
19265                observed
19266                    .lock()
19267                    .expect("heartbeat observations")
19268                    .push(observation.clone());
19269            });
19270
19271        worker
19272            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
19273            .await
19274            .expect("retryable heartbeat failure must remain bounded and recover");
19275
19276        let observations = observations.lock().expect("heartbeat observations");
19277        assert!(observations.len() >= 3, "heartbeat retry must recover");
19278        assert!(
19279            observations.windows(2).all(|pair| {
19280                pair[1]
19281                    .acknowledged_at_unix_millis
19282                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
19283                    >= 850
19284            }),
19285            "a successful retry must start a fresh advertised cadence: {observations:?}"
19286        );
19287        assert_eq!(
19288            server.request_count("/api/worker/heartbeat"),
19289            observations.len() + 1,
19290            "one retryable failure must add exactly one bounded request"
19291        );
19292    }
19293
19294    #[tokio::test]
19295    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
19296        let server = MockWorkerServer::waiting_query_worker();
19297        let client = Client::builder(server.base_url())
19298            .timeout(Duration::from_secs(2))
19299            .build()
19300            .expect("client");
19301        let observations = Arc::new(Mutex::new(Vec::new()));
19302        let observed = Arc::clone(&observations);
19303        let mut worker = Worker::new(client, "rust-snapshot-workers")
19304            .worker_id("rust-snapshot-worker")
19305            .poll_timeout(Duration::from_millis(10))
19306            .on_worker_heartbeat(move |observation| {
19307                observed
19308                    .lock()
19309                    .expect("heartbeat observations")
19310                    .push(observation.clone());
19311            });
19312
19313        worker.register_workflow("snapshot", |ctx, _input| async move {
19314            ctx.wait_signal("finish").await?;
19315            Ok(json!({"status": "finished"}))
19316        });
19317        worker.register_query("snapshot", "current", |ctx, _args| async move {
19318            let current = ctx
19319                .signals("increment")
19320                .iter()
19321                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
19322                .sum::<i64>();
19323            Ok(json!(current))
19324        });
19325        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
19326
19327        worker
19328            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
19329            .await
19330            .expect("pending workflow and query poller must remain live until shutdown");
19331
19332        assert!(
19333            observations.lock().expect("heartbeat observations").len() >= 4,
19334            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
19335        );
19336        assert!(
19337            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
19338            "workflow polling must continue after empty replay acknowledgements"
19339        );
19340        assert!(
19341            server.request_count("/api/worker/query-tasks/poll") >= 2,
19342            "query polling must continue after serving the current query"
19343        );
19344        assert_eq!(
19345            server.request_body("/api/worker/register")["capabilities"],
19346            json!([QUERY_TASKS_CAPABILITY, WORKFLOW_UPDATES_CAPABILITY])
19347        );
19348        assert_eq!(
19349            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
19350            json!({
19351                "queries": ["current"],
19352                "updates": ["replace"],
19353                "update_validators": [],
19354            })
19355        );
19356
19357        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
19358        assert_eq!(
19359            opened["commands"],
19360            json!([{
19361                "type": "open_signal_wait",
19362                "signal_name": "finish",
19363            }])
19364        );
19365
19366        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
19367            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
19368            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
19369            let failure = server.request_body(&fail_path);
19370            assert_eq!(
19371                failure["failure"]["type"],
19372                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
19373            );
19374            assert_eq!(server.request_count(&completion_path), 0);
19375        }
19376
19377        let query_completion =
19378            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
19379        assert_eq!(query_completion["result"], json!(8));
19380
19381        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
19382        assert_eq!(
19383            server.request_count(terminal_path),
19384            1,
19385            "the matching signal must settle the workflow exactly once"
19386        );
19387        let terminal = server.request_body(terminal_path);
19388        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
19389        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
19390        assert_eq!(
19391            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
19392                .expect("terminal workflow result"),
19393            json!({"status": "finished"})
19394        );
19395    }
19396
19397    #[tokio::test]
19398    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
19399        let server = MockWorkerServer::transient_worker_failures();
19400        let client = Client::builder(server.base_url())
19401            .timeout(Duration::from_secs(2))
19402            .build()
19403            .expect("client");
19404        let mut worker = Worker::new(client, "rust-workers")
19405            .worker_id("retry-worker")
19406            .poll_timeout(Duration::from_millis(10))
19407            .retry_policy(WorkerRetryPolicy {
19408                max_retries: 2,
19409                initial_backoff: Duration::from_millis(1),
19410                max_backoff: Duration::from_millis(1),
19411            });
19412        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19413        worker.register_activity(
19414            "counter.activity",
19415            |_ctx, _input| async move { Ok(Value::Null) },
19416        );
19417        worker.register_query(
19418            "counter",
19419            "current",
19420            |_ctx, _args| async move { Ok(json!(8)) },
19421        );
19422
19423        worker
19424            .run_until(tokio::time::sleep(Duration::from_millis(75)))
19425            .await
19426            .expect("transient failures must not stop the worker");
19427
19428        for path in [
19429            "/api/worker/heartbeat",
19430            "/api/worker/workflow-tasks/poll",
19431            "/api/worker/activity-tasks/poll",
19432            "/api/worker/query-tasks/poll",
19433        ] {
19434            assert!(
19435                server.request_count(path) >= 2,
19436                "{path} must continue after its transient failure"
19437            );
19438        }
19439    }
19440
19441    #[tokio::test]
19442    async fn worker_bounds_transport_retries() {
19443        let server = MockWorkerServer::unavailable_polls();
19444        let client = Client::builder(server.base_url())
19445            .timeout(Duration::from_secs(2))
19446            .build()
19447            .expect("client");
19448        let mut worker = Worker::new(client, "rust-workers")
19449            .worker_id("bounded-retry-worker")
19450            .poll_timeout(Duration::from_millis(10))
19451            .retry_policy(WorkerRetryPolicy {
19452                max_retries: 2,
19453                initial_backoff: Duration::from_millis(1),
19454                max_backoff: Duration::from_millis(1),
19455            });
19456        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19457
19458        let error = worker.run().await.expect_err("retry bound must terminate");
19459        assert!(matches!(error, Error::Transport(_)));
19460        assert_eq!(
19461            server.request_count("/api/worker/workflow-tasks/poll"),
19462            3,
19463            "one initial request plus exactly two retries"
19464        );
19465    }
19466
19467    #[tokio::test]
19468    async fn worker_retry_policy_can_disable_poll_retries() {
19469        let server = MockWorkerServer::unavailable_polls();
19470        let client = Client::builder(server.base_url())
19471            .timeout(Duration::from_secs(2))
19472            .build()
19473            .expect("client");
19474        let mut worker = Worker::new(client, "rust-workers")
19475            .worker_id("no-retry-worker")
19476            .poll_timeout(Duration::from_millis(10))
19477            .retry_policy(WorkerRetryPolicy {
19478                max_retries: 0,
19479                initial_backoff: Duration::from_millis(1),
19480                max_backoff: Duration::from_millis(1),
19481            });
19482        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19483
19484        let error = worker
19485            .run_once()
19486            .await
19487            .expect_err("disabled retries must return the first transport failure");
19488        assert!(matches!(error, Error::Transport(_)));
19489        assert_eq!(
19490            server.request_count("/api/worker/workflow-tasks/poll"),
19491            1,
19492            "max_retries=0 must send only the initial request"
19493        );
19494    }
19495
19496    #[tokio::test]
19497    async fn worker_does_not_retry_authentication_failures() {
19498        let server = MockWorkerServer::unauthorized_polls();
19499        let client = Client::builder(server.base_url())
19500            .timeout(Duration::from_secs(2))
19501            .build()
19502            .expect("client");
19503        let mut worker = Worker::new(client, "rust-workers")
19504            .worker_id("unauthorized-worker")
19505            .poll_timeout(Duration::from_millis(10));
19506        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
19507
19508        let error = worker
19509            .run()
19510            .await
19511            .expect_err("authentication must terminate");
19512        let Error::Http { status, body } = error else {
19513            panic!("expected stable HTTP authentication error");
19514        };
19515        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
19516        assert!(body.contains("authentication_failed"));
19517        assert_eq!(
19518            server.request_count("/api/worker/workflow-tasks/poll"),
19519            1,
19520            "authentication failures must not be retried"
19521        );
19522    }
19523
19524    #[derive(Clone, Debug)]
19525    struct CapturedRequest {
19526        method: String,
19527        path: String,
19528        authorization: Option<String>,
19529        namespace: Option<String>,
19530        worker_protocol: Option<String>,
19531        control_protocol: Option<String>,
19532        body: String,
19533        received_at: Instant,
19534    }
19535
19536    struct MockWorkerServer {
19537        addr: SocketAddr,
19538        stop: Arc<AtomicBool>,
19539        requests: Arc<Mutex<Vec<CapturedRequest>>>,
19540        thread: Option<thread::JoinHandle<()>>,
19541    }
19542
19543    #[derive(Clone, Copy, Default)]
19544    struct MockWorkerBehavior {
19545        reject_query_protocol: bool,
19546        reject_query_completion: bool,
19547        waiting_query_worker: bool,
19548        decline_registration: bool,
19549        complete_named_signal: bool,
19550        poll_failures_per_path: usize,
19551        heartbeat_failures: usize,
19552        heartbeat_failure_request: Option<usize>,
19553        delayed_heartbeat_request: Option<usize>,
19554        heartbeat_response_delay: Duration,
19555        concurrent_requests: bool,
19556        unauthorized_polls: bool,
19557        reject_registration: bool,
19558        reject_deregistration: bool,
19559        reject_deregistration_protocol: bool,
19560        cancelled_activity: bool,
19561        draining_polls: bool,
19562        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
19563        workflow_completion_status: Option<&'static str>,
19564        workflow_completion_body: Option<&'static str>,
19565    }
19566
19567    impl MockWorkerServer {
19568        fn start() -> Self {
19569            Self::start_with_behavior(MockWorkerBehavior::default())
19570        }
19571
19572        fn reject_query_protocol() -> Self {
19573            Self::start_with_behavior(MockWorkerBehavior {
19574                reject_query_protocol: true,
19575                ..MockWorkerBehavior::default()
19576            })
19577        }
19578
19579        fn reject_query_completion() -> Self {
19580            Self::start_with_behavior(MockWorkerBehavior {
19581                reject_query_completion: true,
19582                ..MockWorkerBehavior::default()
19583            })
19584        }
19585
19586        fn waiting_query_worker() -> Self {
19587            Self::start_with_behavior(MockWorkerBehavior {
19588                waiting_query_worker: true,
19589                complete_named_signal: true,
19590                ..MockWorkerBehavior::default()
19591            })
19592        }
19593
19594        fn transient_worker_failures() -> Self {
19595            Self::start_with_behavior(MockWorkerBehavior {
19596                poll_failures_per_path: 1,
19597                heartbeat_failures: 1,
19598                ..MockWorkerBehavior::default()
19599            })
19600        }
19601
19602        fn consecutive_poll_failures(count: usize) -> Self {
19603            Self::start_with_behavior(MockWorkerBehavior {
19604                poll_failures_per_path: count,
19605                ..MockWorkerBehavior::default()
19606            })
19607        }
19608
19609        fn delayed_heartbeat_worker() -> Self {
19610            Self::start_with_behavior(MockWorkerBehavior {
19611                waiting_query_worker: true,
19612                delayed_heartbeat_request: Some(2),
19613                heartbeat_response_delay: Duration::from_millis(1_500),
19614                concurrent_requests: true,
19615                cancelled_activity: true,
19616                ..MockWorkerBehavior::default()
19617            })
19618        }
19619
19620        fn heartbeat_retry_worker() -> Self {
19621            Self::start_with_behavior(MockWorkerBehavior {
19622                waiting_query_worker: true,
19623                heartbeat_failure_request: Some(2),
19624                concurrent_requests: true,
19625                ..MockWorkerBehavior::default()
19626            })
19627        }
19628
19629        fn unavailable_polls() -> Self {
19630            Self::start_with_behavior(MockWorkerBehavior {
19631                poll_failures_per_path: usize::MAX,
19632                ..MockWorkerBehavior::default()
19633            })
19634        }
19635
19636        fn unauthorized_polls() -> Self {
19637            Self::start_with_behavior(MockWorkerBehavior {
19638                unauthorized_polls: true,
19639                ..MockWorkerBehavior::default()
19640            })
19641        }
19642
19643        fn rejected_registration() -> Self {
19644            Self::start_with_behavior(MockWorkerBehavior {
19645                reject_registration: true,
19646                ..MockWorkerBehavior::default()
19647            })
19648        }
19649
19650        fn declined_registration() -> Self {
19651            Self::start_with_behavior(MockWorkerBehavior {
19652                decline_registration: true,
19653                ..MockWorkerBehavior::default()
19654            })
19655        }
19656
19657        fn rejected_deregistration() -> Self {
19658            Self::start_with_behavior(MockWorkerBehavior {
19659                reject_deregistration: true,
19660                ..MockWorkerBehavior::default()
19661            })
19662        }
19663
19664        fn rejected_deregistration_protocol() -> Self {
19665            Self::start_with_behavior(MockWorkerBehavior {
19666                reject_deregistration_protocol: true,
19667                ..MockWorkerBehavior::default()
19668            })
19669        }
19670
19671        fn unauthorized_polls_and_rejected_deregistration() -> Self {
19672            Self::start_with_behavior(MockWorkerBehavior {
19673                unauthorized_polls: true,
19674                reject_deregistration: true,
19675                ..MockWorkerBehavior::default()
19676            })
19677        }
19678
19679        fn cancelled_activity() -> Self {
19680            Self::start_with_behavior(MockWorkerBehavior {
19681                cancelled_activity: true,
19682                ..MockWorkerBehavior::default()
19683            })
19684        }
19685
19686        fn draining_polls() -> Self {
19687            Self::start_with_behavior(MockWorkerBehavior {
19688                draining_polls: true,
19689                ..MockWorkerBehavior::default()
19690            })
19691        }
19692
19693        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
19694            Self::start_with_behavior(MockWorkerBehavior {
19695                invalid_task_payload_codec: Some(codec),
19696                ..MockWorkerBehavior::default()
19697            })
19698        }
19699
19700        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
19701            Self::start_with_behavior(MockWorkerBehavior {
19702                workflow_completion_status: Some(status),
19703                workflow_completion_body: Some(body),
19704                ..MockWorkerBehavior::default()
19705            })
19706        }
19707
19708        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
19709            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
19710            listener
19711                .set_nonblocking(true)
19712                .expect("configure mock listener");
19713            let addr = listener.local_addr().expect("mock server address");
19714            let stop = Arc::new(AtomicBool::new(false));
19715            let server_stop = Arc::clone(&stop);
19716            let requests = Arc::new(Mutex::new(Vec::new()));
19717            let server_requests = Arc::clone(&requests);
19718            let thread = thread::spawn(move || {
19719                let mut request_threads = Vec::new();
19720                while !server_stop.load(Ordering::SeqCst) {
19721                    match listener.accept() {
19722                        Ok((mut stream, _)) => {
19723                            if behavior.concurrent_requests {
19724                                let requests = Arc::clone(&server_requests);
19725                                request_threads.push(thread::spawn(move || {
19726                                    handle_mock_worker_request(&mut stream, &requests, behavior)
19727                                }));
19728                            } else {
19729                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
19730                            }
19731                        }
19732                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
19733                            let mut index = 0;
19734                            while index < request_threads.len() {
19735                                if request_threads[index].is_finished() {
19736                                    request_threads
19737                                        .swap_remove(index)
19738                                        .join()
19739                                        .expect("join mock request");
19740                                } else {
19741                                    index += 1;
19742                                }
19743                            }
19744                            thread::sleep(Duration::from_millis(5));
19745                        }
19746                        Err(_) => break,
19747                    }
19748                }
19749                for request_thread in request_threads {
19750                    request_thread.join().expect("join mock request");
19751                }
19752            });
19753
19754            Self {
19755                addr,
19756                stop,
19757                requests,
19758                thread: Some(thread),
19759            }
19760        }
19761
19762        fn base_url(&self) -> String {
19763            format!("http://{}", self.addr)
19764        }
19765
19766        fn worker_protocol_for(&self, path: &str) -> Option<String> {
19767            self.requests
19768                .lock()
19769                .expect("captured requests")
19770                .iter()
19771                .find(|request| request.path == path)
19772                .and_then(|request| request.worker_protocol.clone())
19773        }
19774
19775        fn control_protocol_for(&self, path: &str) -> Option<String> {
19776            self.requests
19777                .lock()
19778                .expect("captured requests")
19779                .iter()
19780                .find(|request| request.path == path)
19781                .and_then(|request| request.control_protocol.clone())
19782        }
19783
19784        fn method_for(&self, path: &str) -> Option<String> {
19785            self.requests
19786                .lock()
19787                .expect("captured requests")
19788                .iter()
19789                .find(|request| request.path == path)
19790                .map(|request| request.method.clone())
19791        }
19792
19793        fn authorization_for(&self, path: &str) -> Option<String> {
19794            self.requests
19795                .lock()
19796                .expect("captured requests")
19797                .iter()
19798                .find(|request| request.path == path)
19799                .and_then(|request| request.authorization.clone())
19800        }
19801
19802        fn namespace_for(&self, path: &str) -> Option<String> {
19803            self.requests
19804                .lock()
19805                .expect("captured requests")
19806                .iter()
19807                .find(|request| request.path == path)
19808                .and_then(|request| request.namespace.clone())
19809        }
19810
19811        fn request_count(&self, path: &str) -> usize {
19812            self.requests
19813                .lock()
19814                .expect("captured requests")
19815                .iter()
19816                .filter(|request| request.path == path)
19817                .count()
19818        }
19819
19820        fn captured_paths(&self) -> Vec<String> {
19821            self.requests
19822                .lock()
19823                .expect("captured requests")
19824                .iter()
19825                .map(|request| request.path.clone())
19826                .collect()
19827        }
19828
19829        fn request_times(&self, path: &str) -> Vec<Instant> {
19830            self.requests
19831                .lock()
19832                .expect("captured requests")
19833                .iter()
19834                .filter(|request| request.path == path)
19835                .map(|request| request.received_at)
19836                .collect()
19837        }
19838
19839        fn request_body(&self, path: &str) -> Value {
19840            let requests = self.requests.lock().expect("captured requests");
19841            let body = &requests
19842                .iter()
19843                .find(|request| request.path == path)
19844                .unwrap_or_else(|| panic!("missing request for {path}"))
19845                .body;
19846            serde_json::from_str(body).unwrap_or_else(|error| {
19847                panic!("invalid JSON request body for {path}: {error}: {body:?}")
19848            })
19849        }
19850
19851        fn request_bodies(&self, path: &str) -> Vec<Value> {
19852            self.requests
19853                .lock()
19854                .expect("captured requests")
19855                .iter()
19856                .filter(|request| request.path == path)
19857                .map(|request| {
19858                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
19859                        panic!(
19860                            "invalid JSON request body for {path}: {error}: {:?}",
19861                            request.body
19862                        )
19863                    })
19864                })
19865                .collect()
19866        }
19867    }
19868
19869    impl Drop for MockWorkerServer {
19870        fn drop(&mut self) {
19871            self.stop.store(true, Ordering::SeqCst);
19872            let _ = TcpStream::connect(self.addr);
19873
19874            if let Some(thread) = self.thread.take() {
19875                thread.join().expect("join mock server");
19876            }
19877        }
19878    }
19879
19880    fn handle_mock_worker_request(
19881        stream: &mut TcpStream,
19882        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
19883        behavior: MockWorkerBehavior,
19884    ) {
19885        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
19886        let mut buffer = [0_u8; 8192];
19887        let mut request = Vec::new();
19888
19889        loop {
19890            match stream.read(&mut buffer) {
19891                Ok(0) => break,
19892                Ok(read) => {
19893                    request.extend_from_slice(&buffer[..read]);
19894                    if mock_request_is_complete(&request) {
19895                        break;
19896                    }
19897                }
19898                Err(error)
19899                    if matches!(
19900                        error.kind(),
19901                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
19902                    ) =>
19903                {
19904                    break;
19905                }
19906                Err(_) => return,
19907            }
19908        }
19909
19910        let request = String::from_utf8_lossy(&request);
19911        let body = request
19912            .split_once("\r\n\r\n")
19913            .map(|(_, body)| body)
19914            .unwrap_or_default();
19915        let path = request
19916            .lines()
19917            .next()
19918            .and_then(|line| line.split_whitespace().nth(1))
19919            .unwrap_or_default();
19920        let method = request
19921            .lines()
19922            .next()
19923            .and_then(|line| line.split_whitespace().next())
19924            .unwrap_or_default();
19925        let authorization = request.lines().find_map(|line| {
19926            let (name, value) = line.split_once(':')?;
19927            name.eq_ignore_ascii_case("Authorization")
19928                .then(|| value.trim().to_string())
19929        });
19930        let namespace = request.lines().find_map(|line| {
19931            let (name, value) = line.split_once(':')?;
19932            name.eq_ignore_ascii_case("X-Namespace")
19933                .then(|| value.trim().to_string())
19934        });
19935        let worker_protocol = request.lines().find_map(|line| {
19936            let (name, value) = line.split_once(':')?;
19937            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
19938                .then(|| value.trim().to_string())
19939        });
19940        let control_protocol = request.lines().find_map(|line| {
19941            let (name, value) = line.split_once(':')?;
19942            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
19943                .then(|| value.trim().to_string())
19944        });
19945        let request_number = {
19946            let mut requests = requests.lock().expect("captured requests");
19947            requests.push(CapturedRequest {
19948                method: method.to_string(),
19949                path: path.to_string(),
19950                authorization,
19951                namespace,
19952                worker_protocol: worker_protocol.clone(),
19953                control_protocol,
19954                body: body.to_string(),
19955                received_at: Instant::now(),
19956            });
19957            requests
19958                .iter()
19959                .filter(|request| request.path == path)
19960                .count()
19961        };
19962
19963        if behavior.reject_registration && path == "/api/worker/register" {
19964            write_mock_response(
19965                stream,
19966                "503 Service Unavailable",
19967                r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
19968            );
19969            return;
19970        }
19971
19972        if path.starts_with("/api/worker/registrations/") {
19973            if behavior.reject_deregistration_protocol {
19974                write_mock_response(
19975                    stream,
19976                    "400 Bad Request",
19977                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.15","requested_version":"1.16"}"#,
19978                );
19979            } else if behavior.reject_deregistration {
19980                write_mock_response(
19981                    stream,
19982                    "403 Forbidden",
19983                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
19984                );
19985            } else {
19986                write_mock_response(
19987                    stream,
19988                    "200 OK",
19989                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
19990                );
19991            }
19992            return;
19993        }
19994
19995        let is_poll = matches!(
19996            path,
19997            "/api/worker/workflow-tasks/poll"
19998                | "/api/worker/activity-tasks/poll"
19999                | "/api/worker/query-tasks/poll"
20000        );
20001        if is_poll && request_number <= behavior.poll_failures_per_path {
20002            return;
20003        }
20004        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
20005            return;
20006        }
20007        if path == "/api/worker/heartbeat"
20008            && behavior.heartbeat_failure_request == Some(request_number)
20009        {
20010            return;
20011        }
20012        if path == "/api/worker/heartbeat"
20013            && behavior.delayed_heartbeat_request == Some(request_number)
20014        {
20015            thread::sleep(behavior.heartbeat_response_delay);
20016        }
20017        if behavior.unauthorized_polls && is_poll {
20018            write_mock_response(
20019                stream,
20020                "401 Unauthorized",
20021                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
20022            );
20023            return;
20024        }
20025        if behavior.draining_polls && is_poll {
20026            write_mock_response(
20027                stream,
20028                "409 Conflict",
20029                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
20030            );
20031            return;
20032        }
20033
20034        if let Some(codec_case) = behavior.invalid_task_payload_codec {
20035            if is_poll && request_number == 1 {
20036                let mut task = match path {
20037                    "/api/worker/workflow-tasks/poll" => json!({
20038                        "task_id": "codec-workflow",
20039                        "workflow_type": "codec.workflow",
20040                        "payload_codec": DEFAULT_CODEC,
20041                        "workflow_task_attempt": 1,
20042                        "lease_owner": "codec-worker"
20043                    }),
20044                    "/api/worker/activity-tasks/poll" => json!({
20045                        "task_id": "codec-activity",
20046                        "activity_attempt_id": "codec-activity-attempt",
20047                        "activity_type": "codec.activity",
20048                        "payload_codec": DEFAULT_CODEC,
20049                        "attempt_number": 1,
20050                        "lease_owner": "codec-worker"
20051                    }),
20052                    "/api/worker/query-tasks/poll" => json!({
20053                        "query_task_id": "codec-query",
20054                        "query_task_attempt": 1,
20055                        "workflow_type": "codec.workflow",
20056                        "query_name": "known",
20057                        "payload_codec": DEFAULT_CODEC,
20058                        "lease_owner": "codec-worker"
20059                    }),
20060                    _ => unreachable!("is_poll limits task codec probe paths"),
20061                };
20062                codec_case.apply(&mut task);
20063                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
20064                return;
20065            }
20066
20067            if matches!(
20068                path,
20069                "/api/worker/workflow-tasks/codec-workflow/fail"
20070                    | "/api/worker/activity-tasks/codec-activity/fail"
20071                    | "/api/worker/query-tasks/codec-query/fail"
20072            ) {
20073                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
20074                return;
20075            }
20076        }
20077
20078        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
20079            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
20080            let body = format!(
20081                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
20082            );
20083            write_mock_response(stream, "400 Bad Request", &body);
20084            return;
20085        }
20086
20087        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
20088        {
20089            write_mock_response(
20090                stream,
20091                "409 Conflict",
20092                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
20093            );
20094            return;
20095        }
20096
20097        if behavior.workflow_completion_status.is_some()
20098            && path == "/api/worker/workflow-tasks/poll"
20099            && request_number == 1
20100        {
20101            write_mock_response(
20102                stream,
20103                "200 OK",
20104                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
20105            );
20106            return;
20107        }
20108
20109        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
20110            if let (Some(status), Some(body)) = (
20111                behavior.workflow_completion_status,
20112                behavior.workflow_completion_body,
20113            ) {
20114                write_mock_response(stream, status, body);
20115                return;
20116            }
20117        }
20118
20119        if behavior.waiting_query_worker {
20120            if behavior.complete_named_signal
20121                && path == "/api/worker/workflow-tasks/poll"
20122                && request_number == 1
20123            {
20124                let body = json!({
20125                    "task": {
20126                        "task_id": "snapshot-open",
20127                        "workflow_id": "snapshot-1",
20128                        "run_id": "snapshot-run-1",
20129                        "workflow_type": "snapshot",
20130                        "payload_codec": DEFAULT_CODEC,
20131                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20132                            .expect("Avro workflow arguments"),
20133                        "history_events": [],
20134                        "workflow_task_attempt": 1,
20135                        "lease_owner": "rust-snapshot-worker"
20136                    }
20137                })
20138                .to_string();
20139                write_mock_response(stream, "200 OK", &body);
20140                return;
20141            }
20142
20143            let signal_request = request_number - usize::from(behavior.complete_named_signal);
20144            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
20145            if path == "/api/worker/workflow-tasks/poll"
20146                && signal_request >= 1
20147                && signal_request <= signal_request_limit
20148            {
20149                let finish = behavior.complete_named_signal && signal_request == 3;
20150                let amounts = if signal_request == 1 {
20151                    vec![3]
20152                } else {
20153                    vec![3, 5]
20154                };
20155                let task_id = if signal_request == 1 {
20156                    "snapshot-wait-3"
20157                } else if finish {
20158                    "snapshot-finish"
20159                } else {
20160                    "snapshot-wait-5"
20161                };
20162                let mut history_events = std::iter::once(json!({
20163                    "event_type": "SignalWaitOpened",
20164                    "payload": {"sequence": 1, "signal_name": "finish"}
20165                }))
20166                .chain(amounts.iter().enumerate().map(|(index, amount)| {
20167                    json!({
20168                        "event_type": "SignalReceived",
20169                        "payload": {
20170                            "signal_id": format!("increment-{amount}"),
20171                            "signal_name": "increment",
20172                            "workflow_sequence": index + 2,
20173                            "payload_codec": DEFAULT_CODEC,
20174                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
20175                                .expect("Avro signal envelope")
20176                        }
20177                    })
20178                }))
20179                .collect::<Vec<_>>();
20180                let (resume_id, resume_name, resume_arguments) = if finish {
20181                    history_events.push(json!({
20182                        "event_type": "SignalReceived",
20183                        "payload": {
20184                            "signal_id": "finish",
20185                            "signal_name": "finish",
20186                            "workflow_sequence": 4,
20187                            "payload_codec": DEFAULT_CODEC,
20188                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20189                                .expect("Avro finish signal envelope")
20190                        }
20191                    }));
20192                    (
20193                        "finish".to_string(),
20194                        "finish".to_string(),
20195                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
20196                            .expect("Avro finish resume signal"),
20197                    )
20198                } else {
20199                    let amount = amounts.last().expect("amount");
20200                    (
20201                        format!("increment-{amount}"),
20202                        "increment".to_string(),
20203                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
20204                            .expect("Avro increment resume signal"),
20205                    )
20206                };
20207                let body = json!({
20208                    "task": {
20209                        "task_id": task_id,
20210                        "workflow_id": "snapshot-1",
20211                        "run_id": "snapshot-run-1",
20212                        "workflow_type": "snapshot",
20213                        "payload_codec": DEFAULT_CODEC,
20214                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20215                            .expect("Avro workflow arguments"),
20216                        "history_events": history_events,
20217                        "workflow_task_attempt": 1,
20218                        "workflow_signal_id": resume_id,
20219                        "signal_name": resume_name,
20220                        "signal_arguments": resume_arguments,
20221                        "lease_owner": "rust-snapshot-worker"
20222                    }
20223                })
20224                .to_string();
20225                write_mock_response(stream, "200 OK", &body);
20226                return;
20227            }
20228
20229            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
20230                let history_events = [3, 5]
20231                    .into_iter()
20232                    .enumerate()
20233                    .map(|(index, amount)| {
20234                        json!({
20235                            "event_type": "SignalReceived",
20236                            "payload": {
20237                                "signal_id": format!("increment-{amount}"),
20238                                "signal_name": "increment",
20239                                "workflow_sequence": index + 2,
20240                                "payload_codec": DEFAULT_CODEC,
20241                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
20242                                    .expect("Avro query signal envelope")
20243                            }
20244                        })
20245                    })
20246                    .collect::<Vec<_>>();
20247                let body = json!({
20248                    "task": {
20249                        "query_task_id": "snapshot-current",
20250                        "query_task_attempt": 1,
20251                        "lease_owner": "rust-snapshot-worker",
20252                        "workflow_id": "snapshot-1",
20253                        "run_id": "snapshot-run-1",
20254                        "workflow_type": "snapshot",
20255                        "query_name": "current",
20256                        "payload_codec": DEFAULT_CODEC,
20257                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20258                            .expect("Avro workflow arguments"),
20259                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
20260                            .expect("Avro query arguments"),
20261                        "history_events": history_events,
20262                        "run_status": "waiting"
20263                    }
20264                })
20265                .to_string();
20266                write_mock_response(stream, "200 OK", &body);
20267                return;
20268            }
20269
20270            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
20271                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
20272            {
20273                write_mock_response(
20274                    stream,
20275                    "200 OK",
20276                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
20277                );
20278                return;
20279            }
20280
20281            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
20282                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
20283                return;
20284            }
20285
20286            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
20287                write_mock_response(
20288                    stream,
20289                    "200 OK",
20290                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
20291                );
20292                return;
20293            }
20294
20295            if path == "/api/worker/query-tasks/snapshot-current/complete" {
20296                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
20297                return;
20298            }
20299        }
20300
20301        if matches!(
20302            path,
20303            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
20304        ) {
20305            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
20306                .expect("typed mock result");
20307            let body = json!({
20308                "result": typed_fidelity_probe().into_json().expect("result projection"),
20309                "result_envelope": result,
20310            })
20311            .to_string();
20312            write_mock_response(stream, "200 OK", &body);
20313            return;
20314        }
20315
20316        if path == "/api/workflows/typed-1" {
20317            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
20318                .expect("typed mock result");
20319            let body = json!({
20320                "workflow_id": "typed-1",
20321                "run_id": "run-typed-1",
20322                "workflow_type": "typed.echo",
20323                "status": "completed",
20324                "output": typed_fidelity_probe().into_json().expect("output projection"),
20325                "output_envelope": result,
20326            })
20327            .to_string();
20328            write_mock_response(stream, "200 OK", &body);
20329            return;
20330        }
20331
20332        let (status, body) = match path {
20333            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
20334            "/api/workflows" => (
20335                "201 Created",
20336                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
20337            ),
20338            "/api/worker/register" if behavior.decline_registration => (
20339                "200 OK",
20340                r#"{"worker_id":"declined-worker","registered":false}"#,
20341            ),
20342            "/api/worker/register" if behavior.waiting_query_worker => (
20343                "200 OK",
20344                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
20345            ),
20346            "/api/worker/register" => (
20347                "200 OK",
20348                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
20349            ),
20350            "/api/worker/heartbeat" => ("200 OK", "{}"),
20351            "/api/worker/activity-tasks/poll"
20352                if behavior.cancelled_activity && request_number == 1 =>
20353            {
20354                (
20355                    "200 OK",
20356                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
20357                )
20358            }
20359            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
20360                ("200 OK", r#"{"task":null}"#)
20361            }
20362            "/api/worker/query-tasks/poll"
20363                if behavior.reject_query_completion && request_number == 1 =>
20364            {
20365                (
20366                    "200 OK",
20367                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
20368                )
20369            }
20370            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
20371            "/api/worker/query-tasks/query-capture/complete"
20372            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
20373            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
20374                "200 OK",
20375                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
20376            ),
20377            "/api/worker/activity-tasks/activity-cancel/complete" => (
20378                "409 Conflict",
20379                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
20380            ),
20381            "/api/worker/activity-tasks/activity-typed/complete"
20382            | "/api/worker/activity-tasks/activity-typed/fail"
20383            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
20384            "/api/workflows/counter-1/query/current" => (
20385                "200 OK",
20386                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
20387            ),
20388            "/api/workflows/counter-1/query/missing" => (
20389                "404 Not Found",
20390                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
20391            ),
20392            "/api/workflows/wf-lifecycle/cancel" => (
20393                "200 OK",
20394                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
20395            ),
20396            "/api/workflows/wf-lifecycle/terminate" => (
20397                "200 OK",
20398                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
20399            ),
20400            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
20401                "200 OK",
20402                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
20403            ),
20404            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
20405                "200 OK",
20406                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
20407            ),
20408            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
20409            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
20410                "409 Conflict",
20411                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
20412            ),
20413            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
20414                "200 OK",
20415                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
20416            ),
20417            "/api/workflows/wf-cancelled" => (
20418                "200 OK",
20419                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
20420            ),
20421            "/api/workflows/wf-terminated" => (
20422                "200 OK",
20423                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
20424            ),
20425            "/api/workflows/wf-timed-out" => (
20426                "200 OK",
20427                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
20428            ),
20429            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
20430                "200 OK",
20431                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
20432            ),
20433            "/api/workflows/wf-selected" => (
20434                "200 OK",
20435                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
20436            ),
20437            "/api/workflows/wf-selected/runs/run-selected" => (
20438                "200 OK",
20439                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
20440            ),
20441            _ => ("404 Not Found", r#"{"message":"not found"}"#),
20442        };
20443        write_mock_response(stream, status, body);
20444    }
20445
20446    fn mock_request_is_complete(request: &[u8]) -> bool {
20447        let Some(header_end) = request
20448            .windows(4)
20449            .position(|window| window == b"\r\n\r\n")
20450            .map(|position| position + 4)
20451        else {
20452            return false;
20453        };
20454        let headers = String::from_utf8_lossy(&request[..header_end]);
20455        let content_length = headers.lines().find_map(|line| {
20456            let (name, value) = line.split_once(':')?;
20457            name.eq_ignore_ascii_case("content-length")
20458                .then(|| value.trim().parse::<usize>().ok())
20459                .flatten()
20460        });
20461
20462        request.len() >= header_end + content_length.unwrap_or(0)
20463    }
20464
20465    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
20466        let response = format!(
20467            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
20468            body.len()
20469        );
20470
20471        let _ = stream.write_all(response.as_bytes());
20472        let _ = stream.flush();
20473    }
20474}