Skip to main content

durable_workflow/
lib.rs

1#![doc = include_str!("../README.md")]
2
3mod runtime_payloads;
4mod runtime_uploads;
5
6use std::{
7    any::{type_name, Any, TypeId},
8    collections::{BTreeMap, HashMap},
9    future::Future,
10    io::{self, Read},
11    pin::Pin,
12    sync::{
13        atomic::{AtomicBool, Ordering},
14        Arc, Mutex, OnceLock,
15    },
16    task::{Context as TaskContext, Poll},
17    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
18};
19
20use apache_avro::{from_avro_datum, to_avro_datum, types::Value as AvroDatum, Schema};
21use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
22use chrono::DateTime;
23use futures_util::{future::OptionFuture, task::noop_waker_ref};
24use serde::{
25    de::DeserializeOwned,
26    ser::{SerializeMap, SerializeSeq},
27    Deserialize, Deserializer, Serialize, Serializer,
28};
29pub use serde_json::{json, Value};
30use sha2::{Digest, Sha256};
31use thiserror::Error;
32pub use uuid::Uuid;
33
34pub const WORKER_PROTOCOL_VERSION: &str = "1.19";
35/// First additive worker protocol that defines portable worker-affinity features.
36pub const PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION: &str = "1.18";
37pub const CONTROL_PLANE_VERSION: &str = "2";
38pub const DEFAULT_CODEC: &str = "avro";
39pub const SDK_VERSION: &str = concat!("durable-workflow-rust/", env!("CARGO_PKG_VERSION"));
40/// Worker-registration capability for authored condition-wait occurrence identity.
41pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY: &str =
42    "condition_wait_occurrence_identity";
43/// Worker-registration capability for portable memo upserts.
44pub const MEMO_UPSERTS_CAPABILITY: &str = "memo_upserts";
45/// Worker-registration capability for server-routed read-only queries.
46pub const QUERY_TASKS_CAPABILITY: &str = "query_tasks";
47/// Worker-registration capability for canonical typed search attributes.
48pub const TYPED_SEARCH_ATTRIBUTES_CAPABILITY: &str = "typed_search_attributes";
49/// Worker-registration capability for synchronous workflow updates.
50pub const WORKFLOW_UPDATES_CAPABILITY: &str = "workflow_updates";
51/// Worker-registration capability for durable named input streams.
52pub const MESSAGE_STREAMS_CAPABILITY: &str = "message_streams";
53/// Worker-registration capability for persisted first-completion selection.
54pub const DURABLE_SELECTION_CAPABILITY: &str = "durable_selection";
55pub const MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.15";
56pub const MESSAGE_STREAM_SIGNAL: &str = "__durable_workflow_message_stream";
57pub const MESSAGE_STREAM_SCHEMA: &str = "durable-workflow.v2.message-stream.message";
58pub const MESSAGE_STREAM_CURSOR_SCHEMA: &str = "durable-workflow.v2.message-stream.cursor";
59pub const MESSAGE_STREAM_MAX_BATCH: usize = 100;
60/// First additive worker protocol that defines query-task transport.
61pub const QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
62/// First additive worker protocol that defines typed search-attribute upserts.
63pub const SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.8";
64/// First additive worker protocol that defines portable memo upserts.
65pub const MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.14";
66/// First additive worker protocol that preserves declared search-attribute types.
67pub const TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.16";
68/// First additive worker protocol that defines external durable condition waits.
69pub const CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.9";
70/// First additive worker protocol that preserves authored condition-wait occurrences.
71pub const CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.17";
72/// First additive worker protocol that defines durable selection groups.
73pub const DURABLE_SELECTION_MINIMUM_WORKER_PROTOCOL_VERSION: &str = "1.19";
74
75pub fn worker_protocol_supports_message_streams(version: &str) -> bool {
76    let Some((major, minor)) = version.split_once('.') else {
77        return false;
78    };
79    major == "1" && minor.parse::<u64>().is_ok_and(|minor| minor >= 15)
80}
81
82fn validate_user_signal_name(signal_name: &str) -> Result<()> {
83    if signal_name == MESSAGE_STREAM_SIGNAL {
84        return Err(Error::Codec(format!(
85            "signal name {MESSAGE_STREAM_SIGNAL:?} is reserved by the workflow runtime"
86        )));
87    }
88    Ok(())
89}
90
91const MAX_LONG_POLL_TIMEOUT_SECONDS: u64 = 60;
92const WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE: &str =
93    "Workflow task waiting for scheduled history.";
94const WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE: &str = "WorkflowTaskWaitingForHistory";
95const MISSING_TASK_PAYLOAD_CODEC: &str = "\0missing-task-payload-codec";
96const NULL_TASK_PAYLOAD_CODEC: &str = "\0null-task-payload-codec";
97const NON_STRING_TASK_PAYLOAD_CODEC: &str = "\0non-string-task-payload-codec";
98const MAX_MEMO_ENTRIES: usize = 100;
99const MAX_MEMO_VALUE_SIZE_BYTES: usize = 10_240;
100const MAX_MEMO_TOTAL_SIZE_BYTES: usize = 65_536;
101
102const QUERY_TASK_FINAL_REJECTION_REASONS: &[&str] = &[
103    "lease_expired",
104    "query_task_not_found",
105    "query_task_not_leased",
106    "query_task_timed_out",
107];
108
109/// Truthful service-worker manifest for features this SDK currently refuses.
110pub fn portable_worker_affinity_capability_manifest() -> Value {
111    json!({
112        "local_activities": {
113            "supported": false,
114            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
115            "reason": "rust_worker_does_not_execute_record_local_activity",
116        },
117        "worker_sessions": {
118            "supported": false,
119            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
120            "reason": "rust_worker_has_no_typed_session_lifecycle",
121        },
122        "sticky_execution": {
123            "supported": false,
124            "minimum_protocol_version": PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION,
125            "reason": "rust_worker_uses_complete_durable_history_replay",
126        },
127    })
128}
129
130/// Canonical Avro Value schema packaged with the crate and parsed by the runtime.
131pub const AVRO_VALUE_SCHEMA_JSON: &str =
132    include_str!("../schema/durable_workflow.protocol.Value.v1.avsc");
133pub const AVRO_VALUE_SCHEMA_FINGERPRINT_HEX: &str = "e2a33dff55802237";
134pub const AVRO_VALUE_SCHEMA_FINGERPRINT: [u8; 8] = [0xe2, 0xa3, 0x3d, 0xff, 0x55, 0x80, 0x22, 0x37];
135const AVRO_SINGLE_OBJECT_MAGIC: [u8; 2] = [0xc3, 0x01];
136
137static AVRO_VALUE_SCHEMA: OnceLock<std::result::Result<Schema, String>> = OnceLock::new();
138static AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA: OnceLock<std::result::Result<Schema, String>> =
139    OnceLock::new();
140
141#[derive(Clone, Copy)]
142enum RequestProtocol {
143    ControlPlane,
144    Worker(&'static str),
145}
146
147pub type Result<T> = std::result::Result<T, Error>;
148
149#[derive(Debug, Error)]
150pub enum Error {
151    #[error("transport error: {0}")]
152    Transport(#[from] reqwest::Error),
153    #[error(
154        "invalid Durable Workflow base URL: omit the SDK-owned /api suffix and pass the Server or Cloud runtime base URL; the SDK appends /api automatically"
155    )]
156    InvalidBaseUrl,
157    #[error("json error: {0}")]
158    Json(#[from] serde_json::Error),
159    #[error("http {status}: {body}")]
160    Http {
161        status: reqwest::StatusCode,
162        body: String,
163    },
164    #[error("codec error: {0}")]
165    Codec(String),
166    #[error(transparent)]
167    QueryFailed(QueryFailure),
168    #[error(transparent)]
169    Protocol(ProtocolFailure),
170    #[error(transparent)]
171    NonDeterministicReplay(ReplayFailure),
172    #[error(transparent)]
173    ChildWorkflowFailed(ChildWorkflowFailure),
174    #[error(transparent)]
175    ActivityFailed(ActivityFailure),
176    #[error(transparent)]
177    ParallelFailed(ParallelFailure),
178    #[error(transparent)]
179    SagaCompensationFailed(SagaCompensationFailure),
180    #[error(transparent)]
181    InvalidParallelGroup(ParallelGroupError),
182    #[error(transparent)]
183    DurableOperationCancelled(DurableOperationCancelled),
184    #[error(transparent)]
185    WorkflowCancellationRequested(WorkflowCancellationRequested),
186    #[error(transparent)]
187    WorkflowCommandRejected(WorkflowCommandRejection),
188    #[error(transparent)]
189    WorkflowFailed(WorkflowTerminalOutcome),
190    #[error(transparent)]
191    WorkflowCancelled(WorkflowTerminalOutcome),
192    #[error(transparent)]
193    WorkflowTerminated(WorkflowTerminalOutcome),
194    #[error(transparent)]
195    WorkflowTimedOut(WorkflowTerminalOutcome),
196    #[error(transparent)]
197    ActivityTaskRejected(ActivityTaskRejection),
198    #[error("workflow handler {0:?} is not registered")]
199    WorkflowNotRegistered(String),
200    #[error("activity handler {0:?} is not registered")]
201    ActivityNotRegistered(String),
202    #[error(
203        "{handler_kind} handler {handler_name:?} {value_kind} type {rust_type} is incompatible with the fixed Avro Value codec: {message}"
204    )]
205    HandlerType {
206        handler_kind: HandlerKind,
207        handler_name: String,
208        value_kind: HandlerValueKind,
209        rust_type: &'static str,
210        message: String,
211    },
212    #[error("workflow future yielded without emitting a durable command")]
213    WorkflowYieldedWithoutCommand,
214    #[error(
215        "workflow_stream_command_identity_missing: workflow stream authoring requires a non-empty server-provided workflow_command_id"
216    )]
217    MissingWorkflowCommandIdentity,
218    #[error("workflow state lock is poisoned")]
219    WorkflowStatePoisoned,
220    #[error("timer duration is too large for the worker protocol")]
221    TimerDurationOverflow,
222    #[error(transparent)]
223    InvalidConditionWaitOptions(#[from] ConditionWaitOptionsError),
224    #[error(transparent)]
225    InvalidSearchAttributeUpdate(#[from] SearchAttributeUpdateError),
226    #[error("operation timed out")]
227    Timeout,
228    #[error(
229        "missing {role}-plane credentials: configure ClientBuilder::{role}_token or ClientBuilder::token; a {opposite_role}-plane token cannot authorize this request"
230    )]
231    MissingRoleCredentials {
232        role: &'static str,
233        opposite_role: &'static str,
234    },
235    #[error("worker loop error: {0}")]
236    WorkerLoop(String),
237    #[error(
238        "workflow command contract for {workflow_type:?} declares update validators, but this Rust SDK cannot execute synchronous pre-accept update validation"
239    )]
240    UnsupportedUpdateValidators { workflow_type: String },
241    #[error("{primary}; worker deregistration also failed: {deregistration}")]
242    WorkerShutdown {
243        primary: Box<Error>,
244        deregistration: Box<Error>,
245    },
246    #[error("invalid child workflow options: {0}")]
247    InvalidChildWorkflowOptions(String),
248    #[error("invalid workflow memo update: {0}")]
249    InvalidMemoUpdate(String),
250    #[error(
251        "workflow_memo_updates_unavailable: the connected runtime did not advertise workflow memo update support"
252    )]
253    WorkflowMemoUpdatesUnavailable,
254    #[error(transparent)]
255    InvalidActivityOptions(ActivityOptionsError),
256    #[error(transparent)]
257    InvalidContinueAsNewOptions(#[from] ContinueAsNewOptionsError),
258    #[doc(hidden)]
259    #[error("workflow requested continue as new")]
260    ContinueAsNew(ContinueAsNewRequest),
261}
262
263/// Validation failure for a durable condition-wait definition.
264#[derive(Clone, Debug, Error, PartialEq, Eq)]
265pub enum ConditionWaitOptionsError {
266    #[error("condition_key must be non-empty")]
267    EmptyKey,
268    #[error("condition_definition_fingerprint must be non-empty")]
269    EmptyPredicateIdentity,
270    #[error("condition timeout is too large for the worker protocol")]
271    TimeoutOverflow,
272}
273
274/// Stable identity and optional durable timeout for a condition wait.
275///
276/// `predicate_identity` is recorded as the worker protocol's
277/// `condition_definition_fingerprint` and must change whenever predicate
278/// behavior changes. Prefer the [`wait_condition!`] macro when the predicate
279/// is written inline; it derives this identity from the predicate tokens.
280#[derive(Clone, Debug, PartialEq, Eq)]
281pub struct ConditionWaitOptions {
282    condition_key: String,
283    predicate_identity: String,
284    timeout: Option<Duration>,
285}
286
287impl ConditionWaitOptions {
288    pub fn new(condition_key: impl Into<String>, predicate_identity: impl Into<String>) -> Self {
289        Self {
290            condition_key: condition_key.into(),
291            predicate_identity: predicate_identity.into(),
292            timeout: None,
293        }
294    }
295
296    pub fn timeout(mut self, timeout: Duration) -> Self {
297        self.timeout = Some(timeout);
298        self
299    }
300
301    fn validate(
302        &self,
303    ) -> std::result::Result<ValidatedConditionWaitOptions, ConditionWaitOptionsError> {
304        let condition_key = self.condition_key.trim();
305        if condition_key.is_empty() {
306            return Err(ConditionWaitOptionsError::EmptyKey);
307        }
308        let predicate_identity = self.predicate_identity.trim();
309        if predicate_identity.is_empty() {
310            return Err(ConditionWaitOptionsError::EmptyPredicateIdentity);
311        }
312        let timeout_seconds = self
313            .timeout
314            .map(|timeout| {
315                timeout
316                    .as_secs()
317                    .checked_add(u64::from(timeout.subsec_nanos() > 0))
318                    .ok_or(ConditionWaitOptionsError::TimeoutOverflow)
319            })
320            .transpose()?;
321
322        Ok(ValidatedConditionWaitOptions {
323            condition_key: condition_key.to_string(),
324            predicate_identity: predicate_identity.to_string(),
325            timeout_seconds,
326        })
327    }
328}
329
330#[derive(Clone, Debug, PartialEq, Eq)]
331struct ValidatedConditionWaitOptions {
332    condition_key: String,
333    predicate_identity: String,
334    timeout_seconds: Option<u64>,
335}
336
337const CONDITION_WAIT_OCCURRENCE_PREFIX: &str = "rust:condition-wait:";
338
339/// Unambiguous terminal result of a durable condition wait.
340#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
341#[serde(rename_all = "snake_case")]
342pub enum ConditionWaitResult {
343    Satisfied,
344    TimedOut,
345}
346
347impl ConditionWaitResult {
348    pub fn is_satisfied(self) -> bool {
349        self == Self::Satisfied
350    }
351
352    pub fn is_timed_out(self) -> bool {
353        self == Self::TimedOut
354    }
355}
356
357/// Build the stable condition definition identity used by [`wait_condition!`].
358#[doc(hidden)]
359pub fn __condition_definition_fingerprint(source: &str) -> String {
360    let mut digest = Sha256::new();
361    digest.update(b"durable-workflow-rust.wait-condition.v1\0");
362    digest.update(source.as_bytes());
363    format!("sha256:{:x}", digest.finalize())
364}
365
366/// Create a durable condition wait whose predicate definition is fingerprinted
367/// from its inline Rust tokens.
368///
369/// The returned [`ConditionWaitCall`] must be awaited. The timeout form is
370/// `wait_condition!(ctx, "approval", timeout: duration, || predicate)`.
371#[macro_export]
372macro_rules! wait_condition {
373    ($ctx:expr, $key:expr, timeout: $timeout:expr, $predicate:expr $(,)?) => {{
374        $ctx.wait_condition(
375            $crate::ConditionWaitOptions::new(
376                $key,
377                $crate::__condition_definition_fingerprint(concat!(
378                    module_path!(),
379                    "\0",
380                    stringify!($predicate)
381                )),
382            )
383            .timeout($timeout),
384            $predicate,
385        )
386    }};
387    ($ctx:expr, $key:expr, $predicate:expr $(,)?) => {{
388        $ctx.wait_condition(
389            $crate::ConditionWaitOptions::new(
390                $key,
391                $crate::__condition_definition_fingerprint(concat!(
392                    module_path!(),
393                    "\0",
394                    stringify!($predicate)
395                )),
396            ),
397            $predicate,
398        )
399    }};
400}
401
402const MAX_SEARCH_ATTRIBUTES_PER_UPDATE: usize = 100;
403const MAX_SEARCH_ATTRIBUTE_KEY_LENGTH: usize = 64;
404const MAX_SEARCH_ATTRIBUTE_STRING_LENGTH: usize = 2_048;
405const MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH: usize = 255;
406const MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES: usize = 65_536;
407
408/// Validation failure for a typed workflow search-attribute update.
409#[derive(Clone, Debug, Error, PartialEq, Eq)]
410pub enum SearchAttributeUpdateError {
411    #[error("search-attribute update requires at least one attribute")]
412    Empty,
413    #[error("search attribute key {0:?} must be 1-64 URL-safe ASCII characters")]
414    InvalidKey(String),
415    #[error("search-attribute update exceeds the limit of 100 attributes")]
416    TooManyAttributes,
417    #[error("search attribute {key:?} {kind} value exceeds {limit} bytes")]
418    ValueTooLong {
419        key: String,
420        kind: &'static str,
421        limit: usize,
422    },
423    #[error(
424        "search attribute {0:?} must not contain an empty string value; use delete() to remove it"
425    )]
426    EmptyString(String),
427    #[error("search attribute {0:?} has a non-finite float value")]
428    NonFiniteFloat(String),
429    #[error("search attribute {0:?} must use an RFC 3339 datetime with an explicit timezone")]
430    InvalidDateTime(String),
431    #[error("search-attribute update exceeds the 65536-byte protocol limit")]
432    PayloadTooLarge,
433}
434
435/// One public typed search-attribute value.
436#[derive(Clone, Debug, PartialEq)]
437pub enum SearchAttributeValue {
438    String(String),
439    Keyword(String),
440    KeywordList(Vec<String>),
441    Int(i64),
442    Float(f64),
443    Bool(bool),
444    DateTime(String),
445    Delete,
446}
447
448impl SearchAttributeValue {
449    fn type_name(&self) -> Option<&'static str> {
450        match self {
451            Self::String(_) => Some("string"),
452            Self::Keyword(_) => Some("keyword"),
453            Self::KeywordList(_) => Some("keyword_list"),
454            Self::Int(_) => Some("int"),
455            Self::Float(_) => Some("float"),
456            Self::Bool(_) => Some("bool"),
457            Self::DateTime(_) => Some("datetime"),
458            Self::Delete => None,
459        }
460    }
461
462    fn normalized(self, key: &str) -> std::result::Result<Self, SearchAttributeUpdateError> {
463        let normalize_string = |value: String, kind: &'static str, limit: usize| {
464            let value = value.trim().to_string();
465            if value.is_empty() {
466                return Err(SearchAttributeUpdateError::EmptyString(key.to_string()));
467            }
468            if value.len() > limit {
469                return Err(SearchAttributeUpdateError::ValueTooLong {
470                    key: key.to_string(),
471                    kind,
472                    limit,
473                });
474            }
475            Ok(value)
476        };
477
478        match self {
479            Self::String(value) => Ok(Self::String(normalize_string(
480                value,
481                "string",
482                MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
483            )?)),
484            Self::Keyword(value) => Ok(Self::Keyword(normalize_string(
485                value,
486                "keyword",
487                MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
488            )?)),
489            Self::KeywordList(values) => {
490                let values = values
491                    .into_iter()
492                    .map(|value| {
493                        let value = value.trim().to_string();
494                        if value.len() > MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH {
495                            return Err(SearchAttributeUpdateError::ValueTooLong {
496                                key: key.to_string(),
497                                kind: "keyword-list entry",
498                                limit: MAX_SEARCH_ATTRIBUTE_KEYWORD_LENGTH,
499                            });
500                        }
501                        Ok(value)
502                    })
503                    .collect::<std::result::Result<Vec<_>, _>>()?;
504                Ok(Self::KeywordList(values))
505            }
506            Self::Float(value) if !value.is_finite() => {
507                Err(SearchAttributeUpdateError::NonFiniteFloat(key.to_string()))
508            }
509            Self::DateTime(value) => {
510                let value =
511                    normalize_string(value, "datetime", MAX_SEARCH_ATTRIBUTE_STRING_LENGTH)?;
512                if DateTime::parse_from_rfc3339(&value).is_err() {
513                    return Err(SearchAttributeUpdateError::InvalidDateTime(key.to_string()));
514                }
515                Ok(Self::DateTime(value))
516            }
517            value => Ok(value),
518        }
519    }
520
521    fn into_json(self) -> Value {
522        match self {
523            Self::String(value) | Self::Keyword(value) | Self::DateTime(value) => {
524                Value::String(value)
525            }
526            Self::KeywordList(values) => {
527                Value::Array(values.into_iter().map(Value::String).collect())
528            }
529            Self::Int(value) => json!(value),
530            Self::Float(value) => json!(value),
531            Self::Bool(value) => json!(value),
532            Self::Delete => Value::Null,
533        }
534    }
535}
536
537/// Validated typed workflow-side search-attribute mutation.
538#[derive(Clone, Debug, Default, PartialEq)]
539pub struct SearchAttributeUpdate {
540    attributes: BTreeMap<String, SearchAttributeValue>,
541}
542
543impl SearchAttributeUpdate {
544    pub fn new() -> Self {
545        Self::default()
546    }
547
548    pub fn set(
549        mut self,
550        key: impl Into<String>,
551        value: SearchAttributeValue,
552    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
553        let key = key.into();
554        validate_search_attribute_key(&key)?;
555        if !self.attributes.contains_key(&key)
556            && self.attributes.len() >= MAX_SEARCH_ATTRIBUTES_PER_UPDATE
557        {
558            return Err(SearchAttributeUpdateError::TooManyAttributes);
559        }
560        self.attributes.insert(key.clone(), value.normalized(&key)?);
561        self.validate_size()?;
562        Ok(self)
563    }
564
565    pub fn string(
566        self,
567        key: impl Into<String>,
568        value: impl Into<String>,
569    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
570        self.set(key, SearchAttributeValue::String(value.into()))
571    }
572
573    pub fn keyword(
574        self,
575        key: impl Into<String>,
576        value: impl Into<String>,
577    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
578        self.set(key, SearchAttributeValue::Keyword(value.into()))
579    }
580
581    pub fn keyword_list<I, V>(
582        self,
583        key: impl Into<String>,
584        values: I,
585    ) -> std::result::Result<Self, SearchAttributeUpdateError>
586    where
587        I: IntoIterator<Item = V>,
588        V: Into<String>,
589    {
590        self.set(
591            key,
592            SearchAttributeValue::KeywordList(values.into_iter().map(Into::into).collect()),
593        )
594    }
595
596    pub fn int(
597        self,
598        key: impl Into<String>,
599        value: i64,
600    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
601        self.set(key, SearchAttributeValue::Int(value))
602    }
603
604    pub fn float(
605        self,
606        key: impl Into<String>,
607        value: f64,
608    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
609        self.set(key, SearchAttributeValue::Float(value))
610    }
611
612    pub fn bool(
613        self,
614        key: impl Into<String>,
615        value: bool,
616    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
617        self.set(key, SearchAttributeValue::Bool(value))
618    }
619
620    pub fn datetime(
621        self,
622        key: impl Into<String>,
623        value: impl Into<String>,
624    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
625        self.set(key, SearchAttributeValue::DateTime(value.into()))
626    }
627
628    pub fn delete(
629        self,
630        key: impl Into<String>,
631    ) -> std::result::Result<Self, SearchAttributeUpdateError> {
632        self.set(key, SearchAttributeValue::Delete)
633    }
634
635    fn validate_size(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
636        let (attributes, _) = self.clone().into_wire_parts();
637        if serde_json::to_vec(&attributes)
638            .map(|payload| payload.len() > MAX_SEARCH_ATTRIBUTE_UPDATE_BYTES)
639            .unwrap_or(true)
640        {
641            return Err(SearchAttributeUpdateError::PayloadTooLarge);
642        }
643        Ok(())
644    }
645
646    fn into_wire_parts(self) -> (Value, BTreeMap<String, String>) {
647        let mut attributes = serde_json::Map::new();
648        let mut attribute_types = BTreeMap::new();
649        for (key, value) in self.attributes {
650            if let Some(type_name) = value.type_name() {
651                attribute_types.insert(key.clone(), type_name.to_string());
652            }
653            attributes.insert(key, value.into_json());
654        }
655        (Value::Object(attributes), attribute_types)
656    }
657
658    fn validate(&self) -> std::result::Result<(), SearchAttributeUpdateError> {
659        if self.attributes.is_empty() {
660            return Err(SearchAttributeUpdateError::Empty);
661        }
662        self.validate_size()
663    }
664}
665
666fn validate_search_attribute_key(key: &str) -> std::result::Result<(), SearchAttributeUpdateError> {
667    let valid = !key.is_empty()
668        && key.len() <= MAX_SEARCH_ATTRIBUTE_KEY_LENGTH
669        && key
670            .bytes()
671            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b':'));
672    if valid {
673        Ok(())
674    } else {
675        Err(SearchAttributeUpdateError::InvalidKey(key.to_string()))
676    }
677}
678
679/// The registered handler family reported by [`Error::HandlerType`].
680#[derive(Clone, Copy, Debug, PartialEq, Eq)]
681pub enum HandlerKind {
682    Workflow,
683    Activity,
684}
685
686impl std::fmt::Display for HandlerKind {
687    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
688        formatter.write_str(match self {
689            Self::Workflow => "workflow",
690            Self::Activity => "activity",
691        })
692    }
693}
694
695/// Whether a typed handler failed to adapt its input or result.
696#[derive(Clone, Copy, Debug, PartialEq, Eq)]
697pub enum HandlerValueKind {
698    Input,
699    Result,
700}
701
702impl std::fmt::Display for HandlerValueKind {
703    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
704        formatter.write_str(match self {
705            Self::Input => "input",
706            Self::Result => "result",
707        })
708    }
709}
710
711/// The lifecycle command sent to a workflow execution.
712#[derive(Clone, Copy, Debug, PartialEq, Eq)]
713pub enum WorkflowCommandKind {
714    Cancel,
715    Terminate,
716}
717
718impl WorkflowCommandKind {
719    fn as_str(self) -> &'static str {
720        match self {
721            Self::Cancel => "cancel",
722            Self::Terminate => "terminate",
723        }
724    }
725}
726
727/// Optional structured fields for a cancellation or termination request.
728#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
729pub struct WorkflowCommandOptions {
730    #[serde(skip_serializing_if = "Option::is_none")]
731    pub reason: Option<String>,
732    #[serde(skip_serializing_if = "Option::is_none")]
733    pub request_id: Option<String>,
734}
735
736/// Server-enforced timeout policy for a workflow start.
737///
738/// These deadlines are distinct from [`WorkflowResultOptions::timeout`], which
739/// only bounds how long the caller waits. A server deadline produces a terminal
740/// [`Error::WorkflowTimedOut`] outcome whose reason is `execution_timeout` or
741/// `run_timeout`.
742#[derive(Clone, Debug, PartialEq, Eq)]
743pub struct WorkflowStartOptions {
744    pub execution_timeout_seconds: u64,
745    pub run_timeout_seconds: u64,
746}
747
748impl Default for WorkflowStartOptions {
749    fn default() -> Self {
750        Self {
751            execution_timeout_seconds: 3600,
752            run_timeout_seconds: 600,
753        }
754    }
755}
756
757impl WorkflowStartOptions {
758    pub fn new() -> Self {
759        Self::default()
760    }
761
762    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
763        self.execution_timeout_seconds = seconds;
764        self
765    }
766
767    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
768        self.run_timeout_seconds = seconds;
769        self
770    }
771
772    fn validate(&self) -> Result<()> {
773        if self.execution_timeout_seconds == 0 {
774            return Err(Error::Codec(
775                "execution_timeout_seconds must be at least 1".to_string(),
776            ));
777        }
778        if self.run_timeout_seconds == 0 {
779            return Err(Error::Codec(
780                "run_timeout_seconds must be at least 1".to_string(),
781            ));
782        }
783        if self.run_timeout_seconds > self.execution_timeout_seconds {
784            return Err(Error::Codec(
785                "run_timeout_seconds cannot exceed execution_timeout_seconds".to_string(),
786            ));
787        }
788
789        Ok(())
790    }
791}
792
793/// Optional routing overrides for a continue-as-new transition.
794///
795/// Omitted values retain the current workflow type and task queue. Server-owned
796/// instance metadata is not accepted here and is carried by the server.
797#[derive(Clone, Debug, Default, PartialEq, Eq)]
798pub struct ContinueAsNewOptions {
799    pub workflow_type: Option<String>,
800    pub task_queue: Option<String>,
801}
802
803impl ContinueAsNewOptions {
804    pub fn new() -> Self {
805        Self::default()
806    }
807
808    pub fn workflow_type(mut self, workflow_type: impl Into<String>) -> Self {
809        self.workflow_type = Some(workflow_type.into());
810        self
811    }
812
813    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
814        self.task_queue = Some(task_queue.into());
815        self
816    }
817
818    fn validate(&self) -> std::result::Result<(), ContinueAsNewOptionsError> {
819        for (field, value) in [
820            ("workflow_type", self.workflow_type.as_deref()),
821            ("task_queue", self.task_queue.as_deref()),
822        ] {
823            if value.is_some_and(|value| value.trim().is_empty()) {
824                return Err(ContinueAsNewOptionsError {
825                    field,
826                    message: format!("{field} must not be empty"),
827                });
828            }
829        }
830        Ok(())
831    }
832}
833
834/// A stable validation error raised before a continue-as-new command is emitted.
835#[derive(Clone, Debug, Error, PartialEq, Eq)]
836#[error("invalid continue-as-new option {field}: {message}")]
837pub struct ContinueAsNewOptionsError {
838    pub field: &'static str,
839    pub message: String,
840}
841
842/// Public history-budget information attached to the current workflow task.
843#[derive(Clone, Debug, Default, PartialEq, Eq)]
844pub struct WorkflowHistoryBudget {
845    pub event_count: u64,
846    pub size_bytes: Option<u64>,
847    pub continue_as_new_recommended: bool,
848    pub pressure: Option<String>,
849}
850
851#[doc(hidden)]
852#[derive(Clone, Debug)]
853pub struct ContinueAsNewRequest {
854    arguments: AvroValue,
855    options: ContinueAsNewOptions,
856}
857
858impl WorkflowCommandOptions {
859    pub fn new() -> Self {
860        Self::default()
861    }
862
863    pub fn reason(mut self, reason: impl Into<String>) -> Self {
864        self.reason = Some(reason.into());
865        self
866    }
867
868    pub fn request_id(mut self, request_id: impl Into<String>) -> Self {
869        self.request_id = Some(request_id.into());
870        self
871    }
872}
873
874/// The accepted, machine-readable result of a lifecycle command.
875#[derive(Clone, Debug, PartialEq)]
876pub struct WorkflowCommandResult {
877    pub command: WorkflowCommandKind,
878    pub workflow_id: String,
879    pub run_id: Option<String>,
880    pub outcome: Option<String>,
881    pub reason: Option<String>,
882    pub command_status: Option<String>,
883    pub raw: Value,
884}
885
886/// A stable rejection returned by instance- or selected-run lifecycle commands.
887#[derive(Clone, Debug, Error)]
888#[error("workflow {command:?} rejected ({reason}, HTTP {status}): {message}")]
889pub struct WorkflowCommandRejection {
890    pub command: WorkflowCommandKind,
891    pub status: u16,
892    pub reason: String,
893    pub message: String,
894    pub workflow_id: String,
895    pub run_id: Option<String>,
896    pub target_scope: Option<String>,
897    pub body: Value,
898}
899
900/// Stable terminal categories returned by [`WorkflowHandle::result`].
901#[derive(Clone, Copy, Debug, PartialEq, Eq)]
902pub enum WorkflowTerminalKind {
903    Failed,
904    Cancelled,
905    Terminated,
906    TimedOut,
907}
908
909/// A typed terminal workflow outcome with durable identity and failure metadata.
910///
911/// Match the corresponding [`enum@Error`] variant and inspect these fields instead
912/// of parsing its display representation. Fields remain `None` when an older
913/// server did not publish that metadata.
914#[derive(Clone, Debug, Error)]
915#[error("workflow {workflow_id} run {run_id:?} ended as {kind:?} ({reason})")]
916pub struct WorkflowTerminalOutcome {
917    pub kind: WorkflowTerminalKind,
918    pub workflow_id: String,
919    pub run_id: Option<String>,
920    pub reason: String,
921    pub failure_category: Option<String>,
922    pub failure_id: Option<String>,
923    pub exception_type: Option<String>,
924    pub exception_class: Option<String>,
925    pub non_retryable: Option<bool>,
926    pub message: Option<String>,
927    pub exception: Option<Value>,
928    pub raw: Value,
929}
930
931/// A worker-side activity settlement or heartbeat rejected by durable state.
932#[derive(Clone, Debug, Error)]
933#[error("activity task {operation} rejected ({reason}, HTTP {status})")]
934pub struct ActivityTaskRejection {
935    pub operation: String,
936    pub status: u16,
937    pub reason: String,
938    pub task_id: String,
939    pub activity_attempt_id: String,
940    pub cancel_requested: bool,
941    pub can_continue: Option<bool>,
942    pub run_closed_reason: Option<String>,
943    pub body: Value,
944}
945
946/// Stable validation categories for [`ActivityOptions`].
947#[derive(Clone, Copy, Debug, PartialEq, Eq)]
948pub enum ActivityOptionsErrorKind {
949    EmptyTaskQueue,
950    EmptyRetryPolicy,
951    InvalidMaxAttempts,
952    BackoffWithoutRetryBudget,
953    TooManyBackoffIntervals,
954    InvalidBackoffCoefficient,
955    BackoffGenerationTooLarge,
956    BackoffOverflow,
957    EmptyNonRetryableErrorType,
958    TimeoutNotPositive,
959    TimeoutOverflow,
960    TimeoutOrder,
961}
962
963/// A machine-readable activity-options validation failure.
964#[derive(Clone, Debug, Error, PartialEq, Eq)]
965#[error("invalid activity options ({kind:?}, {field:?}): {message}")]
966pub struct ActivityOptionsError {
967    pub kind: ActivityOptionsErrorKind,
968    pub field: Option<&'static str>,
969    pub message: String,
970}
971
972impl ActivityOptionsError {
973    fn new(
974        kind: ActivityOptionsErrorKind,
975        field: Option<&'static str>,
976        message: impl Into<String>,
977    ) -> Self {
978        Self {
979            kind,
980            field,
981            message: message.into(),
982        }
983    }
984}
985
986/// Stable terminal categories returned when an awaited activity does not succeed.
987#[derive(Clone, Copy, Debug, PartialEq, Eq)]
988pub enum ActivityFailureKind {
989    Failed,
990    Cancelled,
991    TimedOut,
992}
993
994/// A stable, machine-readable terminal activity failure.
995///
996/// Match [`Error::ActivityFailed`] and inspect `kind`, `reason`,
997/// `failure_category`, or `timeout_kind`; display text is only diagnostic.
998#[derive(Clone, Debug, Error)]
999#[error("activity failed ({reason}): {message}")]
1000pub struct ActivityFailure {
1001    pub kind: ActivityFailureKind,
1002    pub reason: String,
1003    pub message: String,
1004    pub activity_execution_id: Option<String>,
1005    pub activity_attempt_id: Option<String>,
1006    pub activity_type: Option<String>,
1007    pub activity_class: Option<String>,
1008    pub attempt_number: Option<u64>,
1009    pub failure_id: Option<String>,
1010    pub failure_category: Option<String>,
1011    pub timeout_kind: Option<String>,
1012    pub non_retryable: bool,
1013    pub exception_type: Option<String>,
1014    pub exception_class: Option<String>,
1015    pub code: Option<Value>,
1016    pub exception: Option<Value>,
1017}
1018
1019/// Stable terminal categories returned when an awaited child does not succeed.
1020#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1021pub enum ChildWorkflowFailureKind {
1022    Failed,
1023    Cancelled,
1024    Terminated,
1025}
1026
1027/// A stable, machine-readable child workflow failure delivered to its parent.
1028///
1029/// Match [`Error::ChildWorkflowFailed`] and inspect `reason` or `kind` instead
1030/// of parsing the display message. Child and parent identifiers retain the
1031/// relationship recorded in durable history across worker restarts.
1032#[derive(Clone, Debug, Error)]
1033#[error("child workflow failed ({reason}): {message}")]
1034pub struct ChildWorkflowFailure {
1035    pub kind: ChildWorkflowFailureKind,
1036    pub reason: String,
1037    pub message: String,
1038    pub parent_workflow_id: Option<String>,
1039    pub parent_workflow_run_id: Option<String>,
1040    pub child_workflow_id: Option<String>,
1041    pub child_workflow_run_id: Option<String>,
1042    pub child_workflow_type: Option<String>,
1043    pub failure_id: Option<String>,
1044    pub failure_category: Option<String>,
1045    pub exception_type: Option<String>,
1046    pub exception_class: Option<String>,
1047    pub non_retryable: bool,
1048    pub code: Option<Value>,
1049    pub exception: Option<Value>,
1050}
1051
1052/// The identity of one durable workflow execution.
1053#[derive(Clone, Debug, PartialEq, Eq)]
1054pub struct WorkflowIdentity {
1055    pub workflow_id: Option<String>,
1056    pub run_id: Option<String>,
1057}
1058
1059/// A successful child result together with its durable parent-child identity.
1060#[derive(Clone, Debug, PartialEq)]
1061pub struct ChildWorkflowResult {
1062    pub parent: WorkflowIdentity,
1063    pub child: WorkflowIdentity,
1064    pub child_workflow_type: Option<String>,
1065    pub result: Value,
1066}
1067
1068/// Lossless successful child result for fixed Avro Value workflows.
1069#[derive(Clone, Debug, PartialEq)]
1070pub struct ChildWorkflowAvroResult {
1071    pub parent: WorkflowIdentity,
1072    pub child: WorkflowIdentity,
1073    pub child_workflow_type: Option<String>,
1074    pub result: AvroValue,
1075}
1076
1077/// Stable user-facing identity for one member of a durable selection group.
1078#[derive(Clone, Debug, Deserialize, Hash, PartialEq, Eq, Serialize)]
1079#[serde(untagged)]
1080pub enum SelectionKey {
1081    Index(usize),
1082    Name(String),
1083}
1084
1085impl From<usize> for SelectionKey {
1086    fn from(value: usize) -> Self {
1087        Self::Index(value)
1088    }
1089}
1090
1091impl From<String> for SelectionKey {
1092    fn from(value: String) -> Self {
1093        Self::Name(value)
1094    }
1095}
1096
1097impl From<&str> for SelectionKey {
1098    fn from(value: &str) -> Self {
1099        Self::Name(value.to_string())
1100    }
1101}
1102
1103/// Typed result of explicitly awaiting a cancelled non-winning operation.
1104#[derive(Clone, Debug, Error, PartialEq, Eq)]
1105#[error("selected {operation_kind} operation {operation_identity} was explicitly cancelled")]
1106pub struct DurableOperationCancelled {
1107    pub selection_group_id: String,
1108    pub member_key: SelectionKey,
1109    pub member_index: usize,
1110    pub operation_kind: String,
1111    pub operation_identity: String,
1112}
1113
1114/// Stable identity for one enclosing deterministic parallel group.
1115///
1116/// The same fields are attached to every ordinary activity, timer, or child
1117/// workflow command in the group. Nested leaves carry an outer-to-inner path;
1118/// no Rust-specific wire command is introduced.
1119#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
1120pub struct ParallelGroupMetadata {
1121    pub parallel_group_id: String,
1122    pub parallel_group_kind: String,
1123    pub parallel_group_base_sequence: u64,
1124    pub parallel_group_size: usize,
1125    pub parallel_group_index: usize,
1126    #[serde(default, skip_serializing_if = "Option::is_none")]
1127    pub parallel_group_mode: Option<String>,
1128    #[serde(default, skip_serializing_if = "Option::is_none")]
1129    pub selection_member_key: Option<SelectionKey>,
1130    #[serde(default, skip_serializing_if = "Option::is_none")]
1131    pub selection_member_index: Option<usize>,
1132    #[serde(default, skip_serializing_if = "Option::is_none")]
1133    pub selection_member_base_sequence: Option<u64>,
1134    #[serde(default, skip_serializing_if = "Option::is_none")]
1135    pub selection_member_size: Option<usize>,
1136    #[serde(default, skip_serializing_if = "Option::is_none")]
1137    pub selection_member_kind: Option<String>,
1138}
1139
1140/// One input-ordered result returned by [`WorkflowContext::parallel`].
1141#[derive(Clone, Debug, PartialEq)]
1142pub enum ParallelResult {
1143    Activity(Value),
1144    ChildWorkflow(ChildWorkflowResult),
1145    Timer,
1146    Signal(Vec<Value>),
1147    Condition(ConditionWaitResult),
1148    Group(Vec<ParallelResult>),
1149}
1150
1151/// Lossless fixed-Avro counterpart to [`ParallelResult`].
1152#[derive(Clone, Debug, PartialEq)]
1153pub enum ParallelAvroResult {
1154    Activity(AvroValue),
1155    ChildWorkflow(ChildWorkflowAvroResult),
1156    Timer,
1157    Signal(Vec<AvroValue>),
1158    Condition(ConditionWaitResult),
1159    Group(Vec<ParallelAvroResult>),
1160}
1161
1162impl ParallelAvroResult {
1163    fn into_json_result(self) -> Result<ParallelResult> {
1164        match self {
1165            Self::Activity(value) => Ok(ParallelResult::Activity(value.into_json()?)),
1166            Self::ChildWorkflow(result) => Ok(ParallelResult::ChildWorkflow(ChildWorkflowResult {
1167                parent: result.parent,
1168                child: result.child,
1169                child_workflow_type: result.child_workflow_type,
1170                result: result.result.into_json()?,
1171            })),
1172            Self::Timer => Ok(ParallelResult::Timer),
1173            Self::Signal(values) => Ok(ParallelResult::Signal(
1174                values
1175                    .into_iter()
1176                    .map(AvroValue::into_json)
1177                    .collect::<Result<Vec<_>>>()?,
1178            )),
1179            Self::Condition(result) => Ok(ParallelResult::Condition(result)),
1180            Self::Group(results) => Ok(ParallelResult::Group(
1181                results
1182                    .into_iter()
1183                    .map(Self::into_json_result)
1184                    .collect::<Result<Vec<_>>>()?,
1185            )),
1186        }
1187    }
1188}
1189
1190/// One successful leaf retained when another parallel member failed.
1191#[derive(Clone, Debug, PartialEq)]
1192pub struct ParallelCompletion {
1193    pub member_path: Vec<usize>,
1194    pub result: ParallelResult,
1195}
1196
1197/// A deterministic join failed after some siblings had already completed.
1198///
1199/// `cause` retains the typed activity, child-workflow, cancellation, or codec
1200/// error. `completed` is declaration ordered and contains only durable
1201/// successes observed in the same replay. Late sibling completions can add
1202/// entries on a later replay without changing `member_path` or the selected
1203/// positional failure.
1204#[derive(Debug, Error)]
1205#[error("parallel group {group_id} member {member_path:?} failed: {cause}")]
1206pub struct ParallelFailure {
1207    pub group_id: String,
1208    pub member_path: Vec<usize>,
1209    pub group_path: Vec<ParallelGroupMetadata>,
1210    pub completed: Vec<ParallelCompletion>,
1211    #[source]
1212    pub cause: Box<Error>,
1213}
1214
1215/// Stable validation error returned before an invalid group emits commands.
1216#[derive(Clone, Debug, Error, PartialEq, Eq)]
1217#[error("invalid deterministic parallel group ({reason}): {message}")]
1218pub struct ParallelGroupError {
1219    pub reason: &'static str,
1220    pub member_path: Vec<usize>,
1221    pub message: String,
1222}
1223
1224/// Cooperative workflow cancellation observed at an author-controlled point.
1225#[derive(Clone, Debug, Error, PartialEq, Eq)]
1226#[error("workflow cancellation was requested")]
1227pub struct WorkflowCancellationRequested;
1228
1229/// A forward saga failure followed by a terminal compensation failure.
1230#[derive(Debug, Error)]
1231#[error(
1232    "saga forward execution failed; compensation activity {compensation_activity_type} (registration {compensation_registration_order}) also failed: {compensation_failure}"
1233)]
1234pub struct SagaCompensationFailure {
1235    pub initiating_failure: Box<Error>,
1236    pub compensation_failure: Box<Error>,
1237    pub compensation_activity_type: String,
1238    pub compensation_registration_order: usize,
1239}
1240
1241/// Server behavior when a parent closes while its child is still open.
1242#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
1243pub enum ParentClosePolicy {
1244    #[default]
1245    Abandon,
1246    RequestCancel,
1247    Terminate,
1248}
1249
1250impl ParentClosePolicy {
1251    fn as_str(self) -> &'static str {
1252        match self {
1253            Self::Abandon => "abandon",
1254            Self::RequestCancel => "request_cancel",
1255            Self::Terminate => "terminate",
1256        }
1257    }
1258}
1259
1260/// Durable retry policy for one child workflow invocation.
1261#[derive(Clone, Debug, Default, PartialEq, Eq)]
1262pub struct ChildWorkflowRetryPolicy {
1263    pub max_attempts: Option<u32>,
1264    pub backoff_seconds: Vec<u64>,
1265    pub non_retryable_error_types: Vec<String>,
1266}
1267
1268/// Options recorded with a child-workflow command.
1269///
1270/// The task queue is mandatory so routing is explicit and replay-stable.
1271#[derive(Clone, Debug, PartialEq, Eq)]
1272pub struct ChildWorkflowOptions {
1273    pub task_queue: String,
1274    pub parent_close_policy: ParentClosePolicy,
1275    pub retry_policy: Option<ChildWorkflowRetryPolicy>,
1276    pub execution_timeout_seconds: Option<u64>,
1277    pub run_timeout_seconds: Option<u64>,
1278}
1279
1280impl ChildWorkflowOptions {
1281    pub fn new(task_queue: impl Into<String>) -> Self {
1282        Self {
1283            task_queue: task_queue.into(),
1284            parent_close_policy: ParentClosePolicy::Abandon,
1285            retry_policy: None,
1286            execution_timeout_seconds: None,
1287            run_timeout_seconds: None,
1288        }
1289    }
1290
1291    pub fn parent_close_policy(mut self, policy: ParentClosePolicy) -> Self {
1292        self.parent_close_policy = policy;
1293        self
1294    }
1295
1296    pub fn retry_policy(mut self, policy: ChildWorkflowRetryPolicy) -> Self {
1297        self.retry_policy = Some(policy);
1298        self
1299    }
1300
1301    pub fn execution_timeout_seconds(mut self, seconds: u64) -> Self {
1302        self.execution_timeout_seconds = Some(seconds);
1303        self
1304    }
1305
1306    pub fn run_timeout_seconds(mut self, seconds: u64) -> Self {
1307        self.run_timeout_seconds = Some(seconds);
1308        self
1309    }
1310}
1311
1312/// Backoff intervals for one durable activity retry policy.
1313#[derive(Clone, Debug, PartialEq, Eq)]
1314pub enum ActivityBackoff {
1315    /// Use these intervals between attempts. The server repeats the final
1316    /// interval if the retry budget contains more attempts than entries.
1317    Explicit(Vec<Duration>),
1318    /// Generate one interval for every retry using integer exponential growth.
1319    Exponential {
1320        initial_interval: Duration,
1321        coefficient: u32,
1322        maximum_interval: Option<Duration>,
1323    },
1324}
1325
1326/// Durable server-side retry policy for one activity execution.
1327#[derive(Clone, Debug, Default, PartialEq, Eq)]
1328pub struct ActivityRetryPolicy {
1329    pub max_attempts: Option<u32>,
1330    pub backoff: Option<ActivityBackoff>,
1331    pub non_retryable_error_types: Vec<String>,
1332}
1333
1334impl ActivityRetryPolicy {
1335    /// Start a policy with a finite attempt budget, including the first attempt.
1336    pub fn new(max_attempts: u32) -> Self {
1337        Self {
1338            max_attempts: Some(max_attempts),
1339            ..Self::default()
1340        }
1341    }
1342
1343    pub fn backoff_intervals(mut self, intervals: impl IntoIterator<Item = Duration>) -> Self {
1344        self.backoff = Some(ActivityBackoff::Explicit(intervals.into_iter().collect()));
1345        self
1346    }
1347
1348    pub fn exponential_backoff(
1349        mut self,
1350        initial_interval: Duration,
1351        coefficient: u32,
1352        maximum_interval: Option<Duration>,
1353    ) -> Self {
1354        self.backoff = Some(ActivityBackoff::Exponential {
1355            initial_interval,
1356            coefficient,
1357            maximum_interval,
1358        });
1359        self
1360    }
1361
1362    pub fn non_retryable_error_type(mut self, error_type: impl Into<String>) -> Self {
1363        self.non_retryable_error_types.push(error_type.into());
1364        self
1365    }
1366
1367    pub fn non_retryable_error_types(
1368        mut self,
1369        error_types: impl IntoIterator<Item = impl Into<String>>,
1370    ) -> Self {
1371        self.non_retryable_error_types
1372            .extend(error_types.into_iter().map(Into::into));
1373        self
1374    }
1375}
1376
1377/// Options recorded atomically on one deterministic `schedule_activity` command.
1378///
1379/// Durations are rounded up to whole seconds when encoded, so the server never
1380/// receives a shorter timeout or backoff than the caller requested.
1381#[derive(Clone, Debug, Default, PartialEq, Eq)]
1382pub struct ActivityOptions {
1383    pub task_queue: Option<String>,
1384    pub retry_policy: Option<ActivityRetryPolicy>,
1385    pub start_to_close_timeout: Option<Duration>,
1386    pub schedule_to_start_timeout: Option<Duration>,
1387    pub schedule_to_close_timeout: Option<Duration>,
1388    pub heartbeat_timeout: Option<Duration>,
1389}
1390
1391impl ActivityOptions {
1392    pub fn new() -> Self {
1393        Self::default()
1394    }
1395
1396    pub fn task_queue(mut self, task_queue: impl Into<String>) -> Self {
1397        self.task_queue = Some(task_queue.into());
1398        self
1399    }
1400
1401    pub fn retry_policy(mut self, policy: ActivityRetryPolicy) -> Self {
1402        self.retry_policy = Some(policy);
1403        self
1404    }
1405
1406    pub fn start_to_close_timeout(mut self, timeout: Duration) -> Self {
1407        self.start_to_close_timeout = Some(timeout);
1408        self
1409    }
1410
1411    pub fn schedule_to_start_timeout(mut self, timeout: Duration) -> Self {
1412        self.schedule_to_start_timeout = Some(timeout);
1413        self
1414    }
1415
1416    pub fn schedule_to_close_timeout(mut self, timeout: Duration) -> Self {
1417        self.schedule_to_close_timeout = Some(timeout);
1418        self
1419    }
1420
1421    pub fn heartbeat_timeout(mut self, timeout: Duration) -> Self {
1422        self.heartbeat_timeout = Some(timeout);
1423        self
1424    }
1425
1426    fn validate(&self) -> std::result::Result<ValidatedActivityOptions, ActivityOptionsError> {
1427        if self
1428            .task_queue
1429            .as_deref()
1430            .is_some_and(|queue| queue.trim().is_empty())
1431        {
1432            return Err(ActivityOptionsError::new(
1433                ActivityOptionsErrorKind::EmptyTaskQueue,
1434                Some("task_queue"),
1435                "task_queue must not be empty",
1436            ));
1437        }
1438
1439        for (field, value) in [
1440            ("start_to_close_timeout", self.start_to_close_timeout),
1441            ("schedule_to_start_timeout", self.schedule_to_start_timeout),
1442            ("schedule_to_close_timeout", self.schedule_to_close_timeout),
1443            ("heartbeat_timeout", self.heartbeat_timeout),
1444        ] {
1445            if value.is_some_and(|value| value.is_zero()) {
1446                return Err(ActivityOptionsError::new(
1447                    ActivityOptionsErrorKind::TimeoutNotPositive,
1448                    Some(field),
1449                    format!("{field} must be positive"),
1450                ));
1451            }
1452        }
1453
1454        validate_timeout_order(
1455            "heartbeat_timeout",
1456            self.heartbeat_timeout,
1457            "start_to_close_timeout",
1458            self.start_to_close_timeout,
1459        )?;
1460        validate_timeout_order(
1461            "start_to_close_timeout",
1462            self.start_to_close_timeout,
1463            "schedule_to_close_timeout",
1464            self.schedule_to_close_timeout,
1465        )?;
1466        validate_timeout_order(
1467            "schedule_to_start_timeout",
1468            self.schedule_to_start_timeout,
1469            "schedule_to_close_timeout",
1470            self.schedule_to_close_timeout,
1471        )?;
1472
1473        Ok(ValidatedActivityOptions {
1474            task_queue: self.task_queue.clone(),
1475            retry_policy: self
1476                .retry_policy
1477                .as_ref()
1478                .map(validate_activity_retry_policy)
1479                .transpose()?,
1480            start_to_close_timeout: timeout_seconds(
1481                "start_to_close_timeout",
1482                self.start_to_close_timeout,
1483            )?,
1484            schedule_to_start_timeout: timeout_seconds(
1485                "schedule_to_start_timeout",
1486                self.schedule_to_start_timeout,
1487            )?,
1488            schedule_to_close_timeout: timeout_seconds(
1489                "schedule_to_close_timeout",
1490                self.schedule_to_close_timeout,
1491            )?,
1492            heartbeat_timeout: timeout_seconds("heartbeat_timeout", self.heartbeat_timeout)?,
1493        })
1494    }
1495}
1496
1497/// A deferred durable leaf or nested group for [`WorkflowContext::parallel`].
1498///
1499/// Constructors capture arguments but perform no I/O. The join validates the
1500/// complete tree, attaches the existing parallel-group metadata to every
1501/// ordinary command, schedules all leaves, and then suspends.
1502pub enum ParallelOperation {
1503    Activity {
1504        activity_type: String,
1505        options: ActivityOptions,
1506        arguments: Result<AvroValue>,
1507    },
1508    ChildWorkflow {
1509        workflow_type: String,
1510        options: ChildWorkflowOptions,
1511        arguments: Result<AvroValue>,
1512    },
1513    Timer(Duration),
1514    Signal(String),
1515    Condition {
1516        options: ConditionWaitOptions,
1517        predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
1518    },
1519    Group(Vec<ParallelOperation>),
1520}
1521
1522impl ParallelOperation {
1523    pub fn activity<T: Serialize>(activity_type: impl Into<String>, args: T) -> Self {
1524        Self::activity_with_options(activity_type, ActivityOptions::new(), args)
1525    }
1526
1527    pub fn activity_with_options<T: Serialize>(
1528        activity_type: impl Into<String>,
1529        options: ActivityOptions,
1530        args: T,
1531    ) -> Self {
1532        Self::Activity {
1533            activity_type: activity_type.into(),
1534            options,
1535            arguments: AvroValue::from_serialize(&args),
1536        }
1537    }
1538
1539    pub fn child_workflow<T: Serialize>(
1540        workflow_type: impl Into<String>,
1541        options: ChildWorkflowOptions,
1542        args: T,
1543    ) -> Self {
1544        Self::ChildWorkflow {
1545            workflow_type: workflow_type.into(),
1546            options,
1547            arguments: AvroValue::from_serialize(&args),
1548        }
1549    }
1550
1551    pub fn timer(duration: Duration) -> Self {
1552        Self::Timer(duration)
1553    }
1554
1555    pub fn signal(signal_name: impl Into<String>) -> Self {
1556        Self::Signal(signal_name.into())
1557    }
1558
1559    pub fn condition<F>(options: ConditionWaitOptions, predicate: F) -> Self
1560    where
1561        F: Fn() -> Result<bool> + Send + 'static,
1562    {
1563        Self::Condition {
1564            options,
1565            predicate: Box::new(predicate),
1566        }
1567    }
1568
1569    pub fn group(operations: Vec<ParallelOperation>) -> Self {
1570        Self::Group(operations)
1571    }
1572}
1573
1574#[derive(Clone, Debug)]
1575struct ValidatedActivityOptions {
1576    task_queue: Option<String>,
1577    retry_policy: Option<Value>,
1578    start_to_close_timeout: Option<u64>,
1579    schedule_to_start_timeout: Option<u64>,
1580    schedule_to_close_timeout: Option<u64>,
1581    heartbeat_timeout: Option<u64>,
1582}
1583
1584fn validate_timeout_order(
1585    smaller_name: &'static str,
1586    smaller: Option<Duration>,
1587    larger_name: &'static str,
1588    larger: Option<Duration>,
1589) -> std::result::Result<(), ActivityOptionsError> {
1590    if matches!((smaller, larger), (Some(smaller), Some(larger)) if smaller > larger) {
1591        return Err(ActivityOptionsError::new(
1592            ActivityOptionsErrorKind::TimeoutOrder,
1593            Some(smaller_name),
1594            format!("{smaller_name} must be <= {larger_name}"),
1595        ));
1596    }
1597    Ok(())
1598}
1599
1600fn timeout_seconds(
1601    field: &'static str,
1602    value: Option<Duration>,
1603) -> std::result::Result<Option<u64>, ActivityOptionsError> {
1604    value
1605        .map(|value| {
1606            activity_protocol_seconds(value).ok_or_else(|| {
1607                ActivityOptionsError::new(
1608                    ActivityOptionsErrorKind::TimeoutOverflow,
1609                    Some(field),
1610                    format!("{field} is too large for the worker protocol"),
1611                )
1612            })
1613        })
1614        .transpose()
1615}
1616
1617fn duration_seconds_ceil(value: Duration) -> Option<u64> {
1618    value
1619        .as_secs()
1620        .checked_add(u64::from(value.subsec_nanos() > 0))
1621}
1622
1623fn activity_protocol_seconds(value: Duration) -> Option<u64> {
1624    duration_seconds_ceil(value).filter(|seconds| *seconds <= i64::MAX as u64)
1625}
1626
1627fn validate_activity_retry_policy(
1628    policy: &ActivityRetryPolicy,
1629) -> std::result::Result<Value, ActivityOptionsError> {
1630    if policy.max_attempts.is_none()
1631        && policy.backoff.is_none()
1632        && policy.non_retryable_error_types.is_empty()
1633    {
1634        return Err(ActivityOptionsError::new(
1635            ActivityOptionsErrorKind::EmptyRetryPolicy,
1636            Some("retry_policy"),
1637            "retry_policy must configure at least one field",
1638        ));
1639    }
1640    if policy.max_attempts == Some(0) {
1641        return Err(ActivityOptionsError::new(
1642            ActivityOptionsErrorKind::InvalidMaxAttempts,
1643            Some("retry_policy.max_attempts"),
1644            "max_attempts must be >= 1",
1645        ));
1646    }
1647    if policy
1648        .non_retryable_error_types
1649        .iter()
1650        .any(|error_type| error_type.trim().is_empty())
1651    {
1652        return Err(ActivityOptionsError::new(
1653            ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
1654            Some("retry_policy.non_retryable_error_types"),
1655            "non_retryable_error_types must not contain empty values",
1656        ));
1657    }
1658
1659    let backoff_seconds = match &policy.backoff {
1660        None => None,
1661        Some(backoff) => {
1662            let max_attempts = policy.max_attempts.ok_or_else(|| {
1663                ActivityOptionsError::new(
1664                    ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
1665                    Some("retry_policy.backoff"),
1666                    "backoff requires max_attempts",
1667                )
1668            })?;
1669            let retry_count = max_attempts.saturating_sub(1) as usize;
1670            let intervals = match backoff {
1671                ActivityBackoff::Explicit(intervals) => {
1672                    if intervals.len() > retry_count {
1673                        return Err(ActivityOptionsError::new(
1674                            ActivityOptionsErrorKind::TooManyBackoffIntervals,
1675                            Some("retry_policy.backoff"),
1676                            "backoff interval count must not exceed max_attempts - 1",
1677                        ));
1678                    }
1679                    intervals.clone()
1680                }
1681                ActivityBackoff::Exponential {
1682                    initial_interval,
1683                    coefficient,
1684                    maximum_interval,
1685                } => {
1686                    if *coefficient < 1 {
1687                        return Err(ActivityOptionsError::new(
1688                            ActivityOptionsErrorKind::InvalidBackoffCoefficient,
1689                            Some("retry_policy.backoff.coefficient"),
1690                            "backoff coefficient must be >= 1",
1691                        ));
1692                    }
1693                    if retry_count > 10_000 {
1694                        return Err(ActivityOptionsError::new(
1695                            ActivityOptionsErrorKind::BackoffGenerationTooLarge,
1696                            Some("retry_policy.max_attempts"),
1697                            "generated backoff supports at most 10000 retry intervals",
1698                        ));
1699                    }
1700                    let mut current = *initial_interval;
1701                    let mut intervals = Vec::with_capacity(retry_count);
1702                    for _ in 0..retry_count {
1703                        let interval = maximum_interval
1704                            .map(|maximum| current.min(maximum))
1705                            .unwrap_or(current);
1706                        intervals.push(interval);
1707                        if maximum_interval.is_some_and(|maximum| interval == maximum) {
1708                            break;
1709                        }
1710                        current = current.checked_mul(*coefficient).ok_or_else(|| {
1711                            ActivityOptionsError::new(
1712                                ActivityOptionsErrorKind::BackoffOverflow,
1713                                Some("retry_policy.backoff"),
1714                                "generated backoff interval overflowed",
1715                            )
1716                        })?;
1717                    }
1718                    intervals
1719                }
1720            };
1721            Some(
1722                intervals
1723                    .into_iter()
1724                    .map(|interval| {
1725                        activity_protocol_seconds(interval).ok_or_else(|| {
1726                            ActivityOptionsError::new(
1727                                ActivityOptionsErrorKind::BackoffOverflow,
1728                                Some("retry_policy.backoff"),
1729                                "backoff interval is too large for the worker protocol",
1730                            )
1731                        })
1732                    })
1733                    .collect::<std::result::Result<Vec<_>, _>>()?,
1734            )
1735        }
1736    };
1737
1738    let mut encoded = serde_json::Map::new();
1739    if let Some(max_attempts) = policy.max_attempts {
1740        encoded.insert("max_attempts".to_string(), json!(max_attempts));
1741    }
1742    if let Some(backoff_seconds) = backoff_seconds {
1743        encoded.insert("backoff_seconds".to_string(), json!(backoff_seconds));
1744    }
1745    if !policy.non_retryable_error_types.is_empty() {
1746        let mut canonical_error_types = Vec::new();
1747        for error_type in policy
1748            .non_retryable_error_types
1749            .iter()
1750            .map(|error_type| error_type.trim())
1751        {
1752            if !canonical_error_types.contains(&error_type) {
1753                canonical_error_types.push(error_type);
1754            }
1755        }
1756        encoded.insert(
1757            "non_retryable_error_types".to_string(),
1758            json!(canonical_error_types),
1759        );
1760    }
1761    Ok(Value::Object(encoded))
1762}
1763
1764/// A stable, machine-readable failure raised when workflow code no longer
1765/// reconstructs the durable command stream recorded in history.
1766#[derive(Clone, Debug, Error)]
1767#[error("non-deterministic workflow replay ({reason}) at sequence {sequence:?}: {message}")]
1768pub struct ReplayFailure {
1769    pub reason: String,
1770    pub sequence: Option<u64>,
1771    pub expected: Option<String>,
1772    pub actual: Option<String>,
1773    pub message: String,
1774}
1775
1776impl ReplayFailure {
1777    fn new(
1778        reason: impl Into<String>,
1779        sequence: Option<u64>,
1780        expected: Option<String>,
1781        actual: Option<String>,
1782        message: impl Into<String>,
1783    ) -> Self {
1784        Self {
1785            reason: reason.into(),
1786            sequence,
1787            expected,
1788            actual,
1789            message: message.into(),
1790        }
1791    }
1792}
1793
1794/// A stable, machine-readable workflow query or query-task settlement failure.
1795#[derive(Clone, Debug, Error)]
1796#[error("query failed ({reason}, HTTP {status}): {message}")]
1797pub struct QueryFailure {
1798    pub status: u16,
1799    pub reason: String,
1800    pub message: String,
1801    pub body: Value,
1802}
1803
1804/// A stable failure returned when a server rejects an SDK protocol version.
1805#[derive(Clone, Debug, Error)]
1806#[error("protocol rejected ({reason}, HTTP {status}): {message}")]
1807pub struct ProtocolFailure {
1808    pub status: u16,
1809    pub reason: String,
1810    pub message: String,
1811    pub supported_version: Option<String>,
1812    pub requested_version: Option<String>,
1813    pub body: Value,
1814}
1815
1816#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
1817pub struct PayloadEnvelope {
1818    pub codec: String,
1819    pub blob: String,
1820}
1821
1822impl PayloadEnvelope {
1823    pub fn avro<T: Serialize>(value: &T) -> Result<Self> {
1824        encode_payload(value, DEFAULT_CODEC)
1825    }
1826
1827    /// Encode an explicit typed value, including the bytes branch that JSON
1828    /// serialization cannot represent.
1829    pub fn avro_value(value: &AvroValue) -> Result<Self> {
1830        encode_avro_value(value)
1831    }
1832}
1833
1834/// Native adapter for the fixed language-neutral Avro Value schema.
1835#[derive(Clone, Debug)]
1836pub enum AvroValue {
1837    Null,
1838    Boolean(bool),
1839    Long(i64),
1840    Double(f64),
1841    Bytes(Vec<u8>),
1842    String(String),
1843    Array(Vec<AvroValue>),
1844    Map(BTreeMap<String, AvroValue>),
1845}
1846
1847impl PartialEq for AvroValue {
1848    fn eq(&self, other: &Self) -> bool {
1849        match (self, other) {
1850            (Self::Null, Self::Null) => true,
1851            (Self::Boolean(left), Self::Boolean(right)) => left == right,
1852            (Self::Long(left), Self::Long(right)) => left == right,
1853            (Self::Double(left), Self::Double(right)) => left.to_bits() == right.to_bits(),
1854            (Self::Bytes(left), Self::Bytes(right)) => left == right,
1855            (Self::String(left), Self::String(right)) => left == right,
1856            (Self::Array(left), Self::Array(right)) => left == right,
1857            (Self::Map(left), Self::Map(right)) => left == right,
1858            _ => false,
1859        }
1860    }
1861}
1862
1863impl AvroValue {
1864    fn from_serialize<T: Serialize>(value: &T) -> Result<Self> {
1865        Self::from_serde_value(
1866            serde_value::to_value(value).map_err(|error| {
1867                Error::Codec(format!("could not adapt value for Avro: {error}"))
1868            })?,
1869        )
1870    }
1871
1872    fn from_serde_value(value: serde_value::Value) -> Result<Self> {
1873        use serde_value::Value as SerdeValue;
1874
1875        match value {
1876            SerdeValue::Unit => Ok(Self::Null),
1877            SerdeValue::Bool(value) => Ok(Self::Boolean(value)),
1878            SerdeValue::I8(value) => Ok(Self::Long(i64::from(value))),
1879            SerdeValue::I16(value) => Ok(Self::Long(i64::from(value))),
1880            SerdeValue::I32(value) => Ok(Self::Long(i64::from(value))),
1881            SerdeValue::I64(value) => Ok(Self::Long(value)),
1882            SerdeValue::U8(value) => Ok(Self::Long(i64::from(value))),
1883            SerdeValue::U16(value) => Ok(Self::Long(i64::from(value))),
1884            SerdeValue::U32(value) => Ok(Self::Long(i64::from(value))),
1885            SerdeValue::U64(value) => i64::try_from(value).map(Self::Long).map_err(|_| {
1886                Error::Codec(
1887                    "integer_overflow: Avro Value long must be within signed 64-bit range"
1888                        .to_string(),
1889                )
1890            }),
1891            SerdeValue::F32(value) => Self::finite_double(f64::from(value)),
1892            SerdeValue::F64(value) => Self::finite_double(value),
1893            SerdeValue::Char(value) => Ok(Self::String(value.to_string())),
1894            SerdeValue::String(value) => Ok(Self::String(value)),
1895            SerdeValue::Bytes(value) => Ok(Self::Bytes(value)),
1896            SerdeValue::Option(None) => Ok(Self::Null),
1897            SerdeValue::Option(Some(value)) | SerdeValue::Newtype(value) => {
1898                Self::from_serde_value(*value)
1899            }
1900            SerdeValue::Seq(values) => values
1901                .into_iter()
1902                .map(Self::from_serde_value)
1903                .collect::<Result<Vec<_>>>()
1904                .map(Self::Array),
1905            SerdeValue::Map(values) => values
1906                .into_iter()
1907                .map(|(key, value)| {
1908                    let SerdeValue::String(key) = key else {
1909                        return Err(Error::Codec(
1910                            "invalid_map_key: Avro Value map keys must be strings".to_string(),
1911                        ));
1912                    };
1913
1914                    Ok((key, Self::from_serde_value(value)?))
1915                })
1916                .collect::<Result<BTreeMap<_, _>>>()
1917                .map(Self::Map),
1918        }
1919    }
1920
1921    fn finite_double(value: f64) -> Result<Self> {
1922        if !value.is_finite() {
1923            return Err(Error::Codec(
1924                "non_finite_float: Avro Value doubles must be finite".to_string(),
1925            ));
1926        }
1927
1928        Ok(Self::Double(value))
1929    }
1930
1931    fn into_json(self) -> Result<Value> {
1932        match self {
1933            Self::Null => Ok(Value::Null),
1934            Self::Boolean(value) => Ok(Value::Bool(value)),
1935            Self::Long(value) => Ok(Value::Number(value.into())),
1936            Self::Double(value) => serde_json::Number::from_f64(value)
1937                .map(Value::Number)
1938                .ok_or_else(|| {
1939                    Error::Codec(
1940                        "non_finite_float: decoded Avro Value double is not finite".to_string(),
1941                    )
1942                }),
1943            Self::Bytes(value) => Ok(json!({
1944                "$type": "bytes",
1945                "base64": BASE64.encode(value),
1946            })),
1947            Self::String(value) => Ok(Value::String(value)),
1948            Self::Array(values) => values
1949                .into_iter()
1950                .map(Self::into_json)
1951                .collect::<Result<Vec<_>>>()
1952                .map(Value::Array),
1953            Self::Map(values) => values
1954                .into_iter()
1955                .map(|(key, value)| Ok((key, value.into_json()?)))
1956                .collect::<Result<serde_json::Map<_, _>>>()
1957                .map(Value::Object),
1958        }
1959    }
1960
1961    fn into_serde_value(self) -> serde_value::Value {
1962        use serde_value::Value as SerdeValue;
1963
1964        match self {
1965            Self::Null => SerdeValue::Unit,
1966            Self::Boolean(value) => SerdeValue::Bool(value),
1967            Self::Long(value) => SerdeValue::I64(value),
1968            Self::Double(value) => SerdeValue::F64(value),
1969            Self::Bytes(value) => SerdeValue::Bytes(value),
1970            Self::String(value) => SerdeValue::String(value),
1971            Self::Array(values) => {
1972                SerdeValue::Seq(values.into_iter().map(Self::into_serde_value).collect())
1973            }
1974            Self::Map(values) => SerdeValue::Map(
1975                values
1976                    .into_iter()
1977                    .map(|(key, value)| (SerdeValue::String(key), value.into_serde_value()))
1978                    .collect(),
1979            ),
1980        }
1981    }
1982
1983    pub fn deserialize<T: DeserializeOwned>(self) -> Result<T> {
1984        self.into_serde_value().deserialize_into().map_err(|error| {
1985            Error::Codec(format!(
1986                "avro_value_type_mismatch: could not adapt decoded value: {error}"
1987            ))
1988        })
1989    }
1990}
1991
1992impl Serialize for AvroValue {
1993    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1994    where
1995        S: Serializer,
1996    {
1997        match self {
1998            Self::Null => serializer.serialize_unit(),
1999            Self::Boolean(value) => serializer.serialize_bool(*value),
2000            Self::Long(value) => serializer.serialize_i64(*value),
2001            Self::Double(value) => serializer.serialize_f64(*value),
2002            Self::Bytes(value) => serializer.serialize_bytes(value),
2003            Self::String(value) => serializer.serialize_str(value),
2004            Self::Array(values) => {
2005                let mut sequence = serializer.serialize_seq(Some(values.len()))?;
2006                for value in values {
2007                    sequence.serialize_element(value)?;
2008                }
2009                sequence.end()
2010            }
2011            Self::Map(values) => {
2012                let mut map = serializer.serialize_map(Some(values.len()))?;
2013                for (key, value) in values {
2014                    map.serialize_entry(key, value)?;
2015                }
2016                map.end()
2017            }
2018        }
2019    }
2020}
2021
2022pub fn encode_avro_value(value: &AvroValue) -> Result<PayloadEnvelope> {
2023    let datum = avro_value_to_datum(value)?;
2024    let datum = to_avro_datum(avro_value_ordered_map_encoding_schema()?, datum)
2025        .map_err(|err| Error::Codec(format!("avro_value_encode_failed: {err}")))?;
2026    let mut bytes = Vec::with_capacity(datum.len() + 10);
2027    bytes.extend_from_slice(&AVRO_SINGLE_OBJECT_MAGIC);
2028    bytes.extend_from_slice(&AVRO_VALUE_SCHEMA_FINGERPRINT);
2029    bytes.extend_from_slice(&datum);
2030    Ok(PayloadEnvelope {
2031        codec: DEFAULT_CODEC.to_string(),
2032        blob: BASE64.encode(bytes),
2033    })
2034}
2035
2036pub fn decode_avro_value(envelope: &PayloadEnvelope) -> Result<AvroValue> {
2037    if envelope.codec != DEFAULT_CODEC {
2038        return Err(unsupported_payload_codec(&envelope.codec));
2039    }
2040    decode_avro_value_blob(&envelope.blob)
2041}
2042
2043pub fn encode_payload<T: Serialize>(value: &T, codec: &str) -> Result<PayloadEnvelope> {
2044    let blob = match codec {
2045        DEFAULT_CODEC => encode_avro_value(&AvroValue::from_serialize(value)?)?.blob,
2046        other => return Err(unsupported_payload_codec(other)),
2047    };
2048
2049    Ok(PayloadEnvelope {
2050        codec: codec.to_string(),
2051        blob,
2052    })
2053}
2054
2055pub fn decode_payload<T: DeserializeOwned>(envelope: &PayloadEnvelope) -> Result<T> {
2056    match envelope.codec.as_str() {
2057        DEFAULT_CODEC => decode_avro_value(envelope)?.deserialize(),
2058        other => Err(unsupported_payload_codec(other)),
2059    }
2060}
2061
2062fn handler_type_error<T>(
2063    handler_kind: HandlerKind,
2064    handler_name: &str,
2065    value_kind: HandlerValueKind,
2066    message: impl Into<String>,
2067) -> Error {
2068    Error::HandlerType {
2069        handler_kind,
2070        handler_name: handler_name.to_string(),
2071        value_kind,
2072        rust_type: type_name::<T>(),
2073        message: message.into(),
2074    }
2075}
2076
2077fn decode_handler_input<T: DeserializeOwned>(
2078    arguments: AvroValue,
2079    handler_kind: HandlerKind,
2080    handler_name: &str,
2081) -> Result<T> {
2082    let argument = match arguments {
2083        AvroValue::Array(mut arguments) if arguments.len() == 1 => {
2084            arguments.pop().expect("one typed handler argument")
2085        }
2086        AvroValue::Array(arguments) if arguments.is_empty() => AvroValue::Null,
2087        AvroValue::Array(arguments) => {
2088            return Err(handler_type_error::<T>(
2089                handler_kind,
2090                handler_name,
2091                HandlerValueKind::Input,
2092                format!(
2093                    "typed handlers accept one request value, but the task carried {} arguments",
2094                    arguments.len()
2095                ),
2096            ));
2097        }
2098        argument => argument,
2099    };
2100
2101    argument.deserialize().map_err(|error| {
2102        handler_type_error::<T>(
2103            handler_kind,
2104            handler_name,
2105            HandlerValueKind::Input,
2106            error.to_string(),
2107        )
2108    })
2109}
2110
2111fn encode_handler_result<T: Serialize>(
2112    result: &T,
2113    handler_kind: HandlerKind,
2114    handler_name: &str,
2115) -> Result<AvroValue> {
2116    AvroValue::from_serialize(result).map_err(|error| {
2117        handler_type_error::<T>(
2118            handler_kind,
2119            handler_name,
2120            HandlerValueKind::Result,
2121            error.to_string(),
2122        )
2123    })
2124}
2125
2126fn decode_handler_result<T: DeserializeOwned>(
2127    result: AvroValue,
2128    handler_kind: HandlerKind,
2129    handler_name: &str,
2130) -> Result<T> {
2131    result.deserialize().map_err(|error| {
2132        handler_type_error::<T>(
2133            handler_kind,
2134            handler_name,
2135            HandlerValueKind::Result,
2136            error.to_string(),
2137        )
2138    })
2139}
2140
2141#[cfg(test)]
2142fn encode_value_envelope(value: &Value, codec: &str) -> Result<Value> {
2143    Ok(serde_json::to_value(encode_payload(value, codec)?)?)
2144}
2145
2146fn decode_wire_value(value: &Value, fallback_codec: &str) -> Result<Value> {
2147    validate_payload_codec(fallback_codec)?;
2148
2149    if value.is_null() {
2150        return Ok(Value::Null);
2151    }
2152
2153    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2154        return decode_blob(blob, codec);
2155    }
2156
2157    if let Some(blob) = value.as_str() {
2158        return decode_blob(blob, fallback_codec);
2159    }
2160
2161    Err(untagged_payload_value())
2162}
2163
2164fn encode_typed_envelope(value: &AvroValue, codec: &str) -> Result<Value> {
2165    let envelope = match codec {
2166        DEFAULT_CODEC => encode_avro_value(value)?,
2167        other => return Err(unsupported_payload_codec(other)),
2168    };
2169    Ok(serde_json::to_value(envelope)?)
2170}
2171
2172fn decode_wire_avro_value(value: &Value, fallback_codec: &str) -> Result<AvroValue> {
2173    validate_payload_codec(fallback_codec)?;
2174
2175    if value.is_null() {
2176        return Ok(AvroValue::Null);
2177    }
2178
2179    if let Some((codec, blob)) = payload_envelope_parts(value)? {
2180        validate_payload_codec(codec)?;
2181        return decode_avro_value_blob(blob);
2182    }
2183
2184    if let Some(blob) = value.as_str() {
2185        return match fallback_codec {
2186            DEFAULT_CODEC => decode_avro_value_blob(blob),
2187            other => Err(unsupported_payload_codec(other)),
2188        };
2189    }
2190
2191    Err(untagged_payload_value())
2192}
2193
2194fn normalize_avro_arguments(value: AvroValue) -> AvroValue {
2195    match value {
2196        AvroValue::Null => AvroValue::Array(Vec::new()),
2197        AvroValue::Array(_) => value,
2198        other => AvroValue::Array(vec![other]),
2199    }
2200}
2201
2202fn decode_blob(blob: &str, codec: &str) -> Result<Value> {
2203    match codec {
2204        DEFAULT_CODEC => decode_avro_value_blob(blob)?.into_json(),
2205        other => Err(unsupported_payload_codec(other)),
2206    }
2207}
2208
2209fn validate_payload_codec(codec: &str) -> Result<()> {
2210    match codec {
2211        DEFAULT_CODEC => Ok(()),
2212        MISSING_TASK_PAYLOAD_CODEC => {
2213            Err(invalid_task_payload_codec("task payload_codec is missing"))
2214        }
2215        NULL_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec("task payload_codec is null")),
2216        NON_STRING_TASK_PAYLOAD_CODEC => Err(invalid_task_payload_codec(
2217            "task payload_codec must be a string",
2218        )),
2219        other => Err(unsupported_payload_codec(other)),
2220    }
2221}
2222
2223fn invalid_task_payload_codec(reason: &str) -> Error {
2224    Error::Codec(format!(
2225        "unsupported_payload_codec: {reason}; Durable Workflow 2.0 requires an explicit string payload_codec=\"avro\" before worker task execution"
2226    ))
2227}
2228
2229fn payload_envelope_parts(value: &Value) -> Result<Option<(&str, &str)>> {
2230    let Some(object) = value.as_object() else {
2231        return Ok(None);
2232    };
2233    if !object.contains_key("codec") && !object.contains_key("blob") {
2234        return Ok(None);
2235    }
2236
2237    let codec = object
2238        .get("codec")
2239        .and_then(Value::as_str)
2240        .ok_or_else(invalid_payload_envelope)?;
2241    validate_payload_codec(codec)?;
2242    let blob = object
2243        .get("blob")
2244        .and_then(Value::as_str)
2245        .ok_or_else(invalid_payload_envelope)?;
2246    Ok(Some((codec, blob)))
2247}
2248
2249fn invalid_payload_envelope() -> Error {
2250    Error::Codec(
2251        "invalid_payload_envelope: durable payloads must use an object with string codec=\"avro\" and blob fields"
2252            .to_string(),
2253    )
2254}
2255
2256fn validate_workflow_task_commands(commands: &[Value]) -> Result<()> {
2257    for command in commands {
2258        let Some(command) = command.as_object() else {
2259            continue;
2260        };
2261        let Some(command_type) = command.get("type").and_then(Value::as_str) else {
2262            continue;
2263        };
2264        let Some(payload_field) = workflow_command_payload_field(command_type) else {
2265            continue;
2266        };
2267
2268        if let Some(codec) = command.get("payload_codec") {
2269            let codec = codec.as_str().ok_or_else(invalid_payload_envelope)?;
2270            validate_payload_codec(codec)?;
2271        }
2272
2273        let payload = command
2274            .get(payload_field)
2275            .ok_or_else(invalid_payload_envelope)?;
2276        if runtime_payloads::Reference::parse(payload)?.is_none() {
2277            validate_outbound_payload_envelope(payload)?;
2278        }
2279    }
2280    Ok(())
2281}
2282
2283fn workflow_completion_protocol_version(commands: &[Value]) -> &'static str {
2284    if commands.iter().any(|command| {
2285        command.get("type").and_then(Value::as_str) == Some("open_condition_wait")
2286            && command
2287                .get("condition_wait_occurrence_id")
2288                .and_then(Value::as_str)
2289                .is_some_and(|occurrence_id| !occurrence_id.is_empty())
2290    }) {
2291        CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
2292    } else if commands.iter().any(|command| {
2293        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2294            && command.get("attribute_types").is_some()
2295    }) {
2296        TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
2297    } else if commands
2298        .iter()
2299        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
2300    {
2301        MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
2302    } else if commands
2303        .iter()
2304        .any(|command| command.get("type").and_then(Value::as_str) == Some("open_condition_wait"))
2305    {
2306        CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
2307    } else if commands.iter().any(|command| {
2308        command.get("type").and_then(Value::as_str) == Some("upsert_search_attributes")
2309    }) {
2310        SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
2311    } else {
2312        WORKER_PROTOCOL_VERSION
2313    }
2314}
2315
2316fn workflow_completion_protocol_version_with_message_streams(
2317    commands: &[Value],
2318    has_message_stream_metadata: bool,
2319) -> &'static str {
2320    let command_protocol = workflow_completion_protocol_version(commands);
2321    if has_message_stream_metadata && !worker_protocol_supports_message_streams(command_protocol) {
2322        MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
2323    } else {
2324        command_protocol
2325    }
2326}
2327
2328fn workflow_command_payload_field(command_type: &str) -> Option<&'static str> {
2329    match command_type {
2330        "complete_workflow" | "complete_update" | "record_side_effect" => Some("result"),
2331        "schedule_activity" | "start_child_workflow" | "continue_as_new" => Some("arguments"),
2332        "start_service_operation" => Some("request_payload"),
2333        "upsert_memo" => Some("entries"),
2334        _ => None,
2335    }
2336}
2337
2338fn validate_outbound_payload_envelope(value: &Value) -> Result<()> {
2339    let Some((codec, blob)) = payload_envelope_parts(value)? else {
2340        return Err(untagged_payload_value());
2341    };
2342    validate_payload_codec(codec)?;
2343    decode_avro_value_blob(blob)?;
2344    Ok(())
2345}
2346
2347fn unsupported_payload_codec(codec: &str) -> Error {
2348    Error::Codec(format!(
2349        "unsupported_payload_codec: workflow payload codec {codec:?} is not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2350    ))
2351}
2352
2353fn untagged_payload_value() -> Error {
2354    Error::Codec(
2355        "unsupported_payload_codec: untagged durable payload values are not supported by Durable Workflow 2.0; use codec=\"avro\" with the fixed Avro Value schema and single-object framing. JSON remains the HTTP document transport, not a workflow payload codec"
2356            .to_string(),
2357    )
2358}
2359
2360fn decode_avro_value_blob(blob: &str) -> Result<AvroValue> {
2361    let bytes = BASE64.decode(blob).map_err(|err| {
2362        Error::Codec(format!(
2363            "invalid_payload_framing: expected strict base64 Avro single-object bytes: {err}"
2364        ))
2365    })?;
2366
2367    if serde_json::from_slice::<Value>(&bytes).is_ok() {
2368        return Err(unsupported_payload_codec("json"));
2369    }
2370
2371    if bytes.len() < 10 || bytes[..2] != AVRO_SINGLE_OBJECT_MAGIC {
2372        return Err(Error::Codec(
2373            "invalid_payload_framing: expected Avro single-object magic c301".to_string(),
2374        ));
2375    }
2376
2377    let fingerprint: [u8; 8] = bytes[2..10]
2378        .try_into()
2379        .map_err(|_| Error::Codec("invalid Avro fingerprint length".to_string()))?;
2380    if fingerprint != AVRO_VALUE_SCHEMA_FINGERPRINT {
2381        return Err(Error::Codec(format!(
2382            "unsupported_payload_schema: unknown CRC-64-AVRO fingerprint {}",
2383            fingerprint
2384                .iter()
2385                .map(|byte| format!("{byte:02x}"))
2386                .collect::<String>()
2387        )));
2388    }
2389
2390    let mut datum_reader = StrictAvroDatumReader::new(&bytes[10..]);
2391    // The current fingerprint selects the current immutable schema, so reader
2392    // resolution would only re-walk the same recursive union. Future retained
2393    // writer fingerprints supply a distinct reader schema in this branch.
2394    let datum = from_avro_datum(avro_value_schema()?, &mut datum_reader, None);
2395    if datum_reader.truncated {
2396        return Err(Error::Codec(
2397            "invalid_payload_framing: truncated Avro Value datum".to_string(),
2398        ));
2399    }
2400    let datum = datum.map_err(|err| {
2401        Error::Codec(format!(
2402            "invalid_payload_framing: malformed Avro Value datum: {err}"
2403        ))
2404    })?;
2405    if datum_reader.remaining() != 0 {
2406        return Err(Error::Codec(format!(
2407            "invalid_payload_framing: {} trailing bytes after Avro Value datum",
2408            datum_reader.remaining()
2409        )));
2410    }
2411    avro_value_from_datum(datum)
2412}
2413
2414struct StrictAvroDatumReader<'a> {
2415    bytes: &'a [u8],
2416    offset: usize,
2417    truncated: bool,
2418}
2419
2420impl<'a> StrictAvroDatumReader<'a> {
2421    fn new(bytes: &'a [u8]) -> Self {
2422        Self {
2423            bytes,
2424            offset: 0,
2425            truncated: false,
2426        }
2427    }
2428
2429    fn remaining(&self) -> usize {
2430        self.bytes.len() - self.offset
2431    }
2432}
2433
2434impl Read for StrictAvroDatumReader<'_> {
2435    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
2436        let count = buffer.len().min(self.remaining());
2437        buffer[..count].copy_from_slice(&self.bytes[self.offset..self.offset + count]);
2438        self.offset += count;
2439        if count < buffer.len() {
2440            self.truncated = true;
2441        }
2442
2443        Ok(count)
2444    }
2445}
2446
2447fn avro_value_to_datum(value: &AvroValue) -> Result<AvroDatum> {
2448    let branch = match value {
2449        AvroValue::Null => AvroDatum::Union(0, Box::new(AvroDatum::Null)),
2450        AvroValue::Boolean(value) => AvroDatum::Union(
2451            1,
2452            Box::new(AvroDatum::Record(vec![(
2453                "boolean".to_string(),
2454                AvroDatum::Boolean(*value),
2455            )])),
2456        ),
2457        AvroValue::Long(value) => AvroDatum::Union(
2458            2,
2459            Box::new(AvroDatum::Record(vec![(
2460                "long".to_string(),
2461                AvroDatum::Long(*value),
2462            )])),
2463        ),
2464        AvroValue::Double(value) => {
2465            if !value.is_finite() {
2466                return Err(Error::Codec(
2467                    "non_finite_float: Avro Value doubles must be finite".to_string(),
2468                ));
2469            }
2470            AvroDatum::Union(
2471                3,
2472                Box::new(AvroDatum::Record(vec![(
2473                    "double".to_string(),
2474                    AvroDatum::Double(*value),
2475                )])),
2476            )
2477        }
2478        AvroValue::Bytes(value) => AvroDatum::Union(
2479            4,
2480            Box::new(AvroDatum::Record(vec![(
2481                "bytes".to_string(),
2482                AvroDatum::Bytes(value.clone()),
2483            )])),
2484        ),
2485        AvroValue::String(value) => AvroDatum::Union(
2486            5,
2487            Box::new(AvroDatum::Record(vec![(
2488                "string".to_string(),
2489                AvroDatum::String(value.clone()),
2490            )])),
2491        ),
2492        AvroValue::Array(values) => AvroDatum::Union(
2493            6,
2494            Box::new(AvroDatum::Record(vec![(
2495                "items".to_string(),
2496                AvroDatum::Array(
2497                    values
2498                        .iter()
2499                        .map(avro_value_to_datum)
2500                        .collect::<Result<Vec<_>>>()?,
2501                ),
2502            )])),
2503        ),
2504        AvroValue::Map(values) => AvroDatum::Union(
2505            7,
2506            Box::new(AvroDatum::Record(vec![(
2507                "entries".to_string(),
2508                AvroDatum::Array(
2509                    values
2510                        .iter()
2511                        .map(|(key, value)| {
2512                            Ok(AvroDatum::Record(vec![
2513                                ("key".to_string(), AvroDatum::String(key.clone())),
2514                                ("value".to_string(), avro_value_to_datum(value)?),
2515                            ]))
2516                        })
2517                        .collect::<Result<Vec<_>>>()?,
2518                ),
2519            )])),
2520        ),
2521    };
2522    Ok(AvroDatum::Record(vec![("value".to_string(), branch)]))
2523}
2524
2525fn avro_value_from_datum(datum: AvroDatum) -> Result<AvroValue> {
2526    let AvroDatum::Record(mut outer) = datum else {
2527        return Err(Error::Codec(
2528            "invalid_payload_framing: datum is not a Value record".to_string(),
2529        ));
2530    };
2531    let (_, branch) = outer
2532        .pop()
2533        .filter(|(name, _)| name == "value")
2534        .ok_or_else(|| Error::Codec("invalid_payload_framing: Value field missing".to_string()))?;
2535    let AvroDatum::Union(_, branch) = branch else {
2536        return Err(Error::Codec(
2537            "invalid_payload_framing: invalid Value union".to_string(),
2538        ));
2539    };
2540    match *branch {
2541        AvroDatum::Null => Ok(AvroValue::Null),
2542        AvroDatum::Record(mut fields) => {
2543            let (name, value) = fields.pop().ok_or_else(|| {
2544                Error::Codec("invalid_payload_framing: empty Value branch".to_string())
2545            })?;
2546            match (name.as_str(), value) {
2547                ("boolean", AvroDatum::Boolean(value)) => Ok(AvroValue::Boolean(value)),
2548                ("long", AvroDatum::Long(value)) => Ok(AvroValue::Long(value)),
2549                ("double", AvroDatum::Double(value)) if value.is_finite() => {
2550                    Ok(AvroValue::Double(value))
2551                }
2552                ("bytes", AvroDatum::Bytes(value)) => Ok(AvroValue::Bytes(value)),
2553                ("string", AvroDatum::String(value)) => Ok(AvroValue::String(value)),
2554                ("items", AvroDatum::Array(values)) => values
2555                    .into_iter()
2556                    .map(avro_value_from_datum)
2557                    .collect::<Result<Vec<_>>>()
2558                    .map(AvroValue::Array),
2559                ("entries", AvroDatum::Map(values)) => values
2560                    .into_iter()
2561                    .map(|(key, value)| Ok((key, avro_value_from_datum(value)?)))
2562                    .collect::<Result<BTreeMap<_, _>>>()
2563                    .map(AvroValue::Map),
2564                _ => Err(Error::Codec(
2565                    "invalid_payload_framing: unknown Value branch".to_string(),
2566                )),
2567            }
2568        }
2569        _ => Err(Error::Codec(
2570            "invalid_payload_framing: invalid Value branch".to_string(),
2571        )),
2572    }
2573}
2574
2575fn avro_value_schema() -> Result<&'static Schema> {
2576    match AVRO_VALUE_SCHEMA.get_or_init(|| {
2577        Schema::parse_str(AVRO_VALUE_SCHEMA_JSON)
2578            .map_err(|err| format!("could not parse Avro Value schema: {err}"))
2579    }) {
2580        Ok(schema) => Ok(schema),
2581        Err(message) => Err(Error::Codec(message.clone())),
2582    }
2583}
2584
2585fn avro_value_ordered_map_encoding_schema() -> Result<&'static Schema> {
2586    match AVRO_VALUE_ORDERED_MAP_ENCODING_SCHEMA.get_or_init(|| {
2587        // Apache Avro's Value::Map uses a randomized HashMap. Arrays and maps
2588        // have the same block representation when each ordered array record
2589        // contains the map key followed by its value, so this encoding-only
2590        // adaptation lets the official encoder retain BTreeMap wire order.
2591        let mut schema: Value = serde_json::from_str(AVRO_VALUE_SCHEMA_JSON)
2592            .map_err(|err| format!("could not read packaged Avro Value schema: {err}"))?;
2593        let entries_schema = schema
2594            .pointer_mut("/fields/0/type/7/fields/0/type")
2595            .ok_or_else(|| "packaged Avro Value map schema is missing".to_string())?;
2596        if *entries_schema != json!({"type": "map", "values": "Value"}) {
2597            return Err("packaged Avro Value map schema changed unexpectedly".to_string());
2598        }
2599        *entries_schema = json!({
2600            "type": "array",
2601            "items": {
2602                "type": "record",
2603                "name": "MapEntry",
2604                "fields": [
2605                    {"name": "key", "type": "string"},
2606                    {"name": "value", "type": "Value"}
2607                ]
2608            }
2609        });
2610        Schema::parse_str(&schema.to_string())
2611            .map_err(|err| format!("could not parse ordered-map Avro Value schema: {err}"))
2612    }) {
2613        Ok(schema) => Ok(schema),
2614        Err(message) => Err(Error::Codec(message.clone())),
2615    }
2616}
2617
2618#[derive(Clone, Debug)]
2619pub struct Client {
2620    http: reqwest::Client,
2621    base_url: String,
2622    token: Option<String>,
2623    control_token: Option<String>,
2624    worker_token: Option<String>,
2625    namespace: String,
2626    max_external_payload_bytes: usize,
2627    worker_storage_admission: Option<WorkerStorageAdmission>,
2628    runtime_upload_policy: Arc<Mutex<runtime_uploads::PolicyCache>>,
2629}
2630
2631impl Client {
2632    pub fn new(base_url: impl Into<String>) -> Result<Self> {
2633        Self::builder(base_url).build()
2634    }
2635
2636    pub fn builder(base_url: impl Into<String>) -> ClientBuilder {
2637        ClientBuilder {
2638            base_url: base_url.into(),
2639            token: None,
2640            control_token: None,
2641            worker_token: None,
2642            namespace: "default".to_string(),
2643            timeout: Duration::from_secs(60),
2644            max_external_payload_bytes: 64 * 1024 * 1024,
2645        }
2646    }
2647
2648    pub async fn health(&self) -> Result<Value> {
2649        self.request_json(
2650            reqwest::Method::GET,
2651            "/health",
2652            RequestProtocol::ControlPlane,
2653            Option::<&Value>::None,
2654        )
2655        .await
2656    }
2657
2658    pub async fn cluster_info(&self) -> Result<Value> {
2659        self.request_json(
2660            reqwest::Method::GET,
2661            "/cluster/info",
2662            RequestProtocol::ControlPlane,
2663            Option::<&Value>::None,
2664        )
2665        .await
2666    }
2667
2668    pub async fn start_workflow<T: Serialize>(
2669        &self,
2670        workflow_type: &str,
2671        task_queue: &str,
2672        workflow_id: &str,
2673        input: T,
2674    ) -> Result<WorkflowHandle> {
2675        self.start_workflow_with_options(
2676            workflow_type,
2677            task_queue,
2678            workflow_id,
2679            WorkflowStartOptions::default(),
2680            input,
2681        )
2682        .await
2683    }
2684
2685    /// Start a workflow with explicit server-enforced execution and run
2686    /// deadlines.
2687    pub async fn start_workflow_with_options<T: Serialize>(
2688        &self,
2689        workflow_type: &str,
2690        task_queue: &str,
2691        workflow_id: &str,
2692        options: WorkflowStartOptions,
2693        input: T,
2694    ) -> Result<WorkflowHandle> {
2695        options.validate()?;
2696        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2697        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2698        let body = json!({
2699            "workflow_id": workflow_id,
2700            "workflow_type": workflow_type,
2701            "task_queue": task_queue,
2702            "input": input_envelope,
2703            "execution_timeout_seconds": options.execution_timeout_seconds,
2704            "run_timeout_seconds": options.run_timeout_seconds
2705        });
2706
2707        let data: Value = self
2708            .request_json(
2709                reqwest::Method::POST,
2710                "/workflows",
2711                RequestProtocol::ControlPlane,
2712                Some(&body),
2713            )
2714            .await?;
2715
2716        Ok(WorkflowHandle {
2717            client: self.clone(),
2718            workflow_id: data
2719                .get("workflow_id")
2720                .and_then(Value::as_str)
2721                .unwrap_or(workflow_id)
2722                .to_string(),
2723            run_id: data
2724                .get("run_id")
2725                .and_then(Value::as_str)
2726                .map(str::to_string),
2727            workflow_type: data
2728                .get("workflow_type")
2729                .and_then(Value::as_str)
2730                .unwrap_or(workflow_type)
2731                .to_string(),
2732        })
2733    }
2734
2735    pub async fn signal_workflow<T: Serialize>(
2736        &self,
2737        workflow_id: &str,
2738        signal_name: &str,
2739        input: T,
2740    ) -> Result<Value> {
2741        self.signal_workflow_target(workflow_id, None, signal_name, input)
2742            .await
2743    }
2744
2745    /// Append one idempotently identified item to an instance-scoped input stream.
2746    pub async fn append_message_stream<T: Serialize>(
2747        &self,
2748        workflow_id: &str,
2749        stream_name: &str,
2750        message_id: &str,
2751        input: T,
2752    ) -> Result<Value> {
2753        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2754        let body = json!({
2755            "message_id": message_id,
2756            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?
2757        });
2758        self.request_json(
2759            reqwest::Method::POST,
2760            &format!("/workflows/{workflow_id}/message-streams/{stream_name}/messages"),
2761            RequestProtocol::ControlPlane,
2762            Some(&body),
2763        )
2764        .await
2765    }
2766
2767    /// Signal only if `run_id` is still the current run for this instance.
2768    pub async fn signal_workflow_run<T: Serialize>(
2769        &self,
2770        workflow_id: &str,
2771        run_id: &str,
2772        signal_name: &str,
2773        input: T,
2774    ) -> Result<Value> {
2775        self.signal_workflow_target(workflow_id, Some(run_id), signal_name, input)
2776            .await
2777    }
2778
2779    async fn signal_workflow_target<T: Serialize>(
2780        &self,
2781        workflow_id: &str,
2782        run_id: Option<&str>,
2783        signal_name: &str,
2784        input: T,
2785    ) -> Result<Value> {
2786        validate_user_signal_name(signal_name)?;
2787        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2788        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
2789        let body = json!({
2790            "input": input_envelope
2791        });
2792        let path = match run_id {
2793            Some(run_id) => {
2794                format!("/workflows/{workflow_id}/runs/{run_id}/signal/{signal_name}")
2795            }
2796            None => format!("/workflows/{workflow_id}/signal/{signal_name}"),
2797        };
2798        self.request_json(
2799            reqwest::Method::POST,
2800            &path,
2801            RequestProtocol::ControlPlane,
2802            Some(&body),
2803        )
2804        .await
2805    }
2806
2807    /// Request cooperative cancellation of the current run for an instance.
2808    pub async fn cancel_workflow(
2809        &self,
2810        workflow_id: &str,
2811        options: WorkflowCommandOptions,
2812    ) -> Result<WorkflowCommandResult> {
2813        self.workflow_command(workflow_id, None, WorkflowCommandKind::Cancel, options)
2814            .await
2815    }
2816
2817    /// Request cooperative cancellation only if `run_id` is still current.
2818    pub async fn cancel_workflow_run(
2819        &self,
2820        workflow_id: &str,
2821        run_id: &str,
2822        options: WorkflowCommandOptions,
2823    ) -> Result<WorkflowCommandResult> {
2824        self.workflow_command(
2825            workflow_id,
2826            Some(run_id),
2827            WorkflowCommandKind::Cancel,
2828            options,
2829        )
2830        .await
2831    }
2832
2833    /// Forcefully terminate the current run for an instance.
2834    pub async fn terminate_workflow(
2835        &self,
2836        workflow_id: &str,
2837        options: WorkflowCommandOptions,
2838    ) -> Result<WorkflowCommandResult> {
2839        self.workflow_command(workflow_id, None, WorkflowCommandKind::Terminate, options)
2840            .await
2841    }
2842
2843    /// Forcefully terminate only if `run_id` is still current.
2844    pub async fn terminate_workflow_run(
2845        &self,
2846        workflow_id: &str,
2847        run_id: &str,
2848        options: WorkflowCommandOptions,
2849    ) -> Result<WorkflowCommandResult> {
2850        self.workflow_command(
2851            workflow_id,
2852            Some(run_id),
2853            WorkflowCommandKind::Terminate,
2854            options,
2855        )
2856        .await
2857    }
2858
2859    async fn workflow_command(
2860        &self,
2861        workflow_id: &str,
2862        run_id: Option<&str>,
2863        command: WorkflowCommandKind,
2864        options: WorkflowCommandOptions,
2865    ) -> Result<WorkflowCommandResult> {
2866        let path = match run_id {
2867            Some(run_id) => format!(
2868                "/workflows/{workflow_id}/runs/{run_id}/{}",
2869                command.as_str()
2870            ),
2871            None => format!("/workflows/{workflow_id}/{}", command.as_str()),
2872        };
2873        let data = match self
2874            .request_json(
2875                reqwest::Method::POST,
2876                &path,
2877                RequestProtocol::ControlPlane,
2878                Some(&options),
2879            )
2880            .await
2881        {
2882            Ok(data) => data,
2883            Err(Error::Http { status, body }) => {
2884                return Err(Error::WorkflowCommandRejected(workflow_command_rejection(
2885                    command,
2886                    status,
2887                    body,
2888                    workflow_id,
2889                    run_id,
2890                )));
2891            }
2892            Err(error) => return Err(error),
2893        };
2894
2895        Ok(workflow_command_result(command, data, workflow_id, run_id))
2896    }
2897
2898    /// Execute a named, read-only query against a running or completed workflow.
2899    ///
2900    /// Arguments and results use the platform payload envelope. Server and
2901    /// worker rejections are returned as [`Error::QueryFailed`] with a stable
2902    /// reason, HTTP status, and original response body.
2903    pub async fn query_workflow<T: Serialize>(
2904        &self,
2905        workflow_id: &str,
2906        query_name: &str,
2907        input: T,
2908    ) -> Result<Value> {
2909        self.query_workflow_target(workflow_id, None, query_name, input)
2910            .await
2911    }
2912
2913    /// Query only if `run_id` is still current, preventing accidental retargeting.
2914    pub async fn query_workflow_run<T: Serialize>(
2915        &self,
2916        workflow_id: &str,
2917        run_id: &str,
2918        query_name: &str,
2919        input: T,
2920    ) -> Result<Value> {
2921        self.query_workflow_target(workflow_id, Some(run_id), query_name, input)
2922            .await
2923    }
2924
2925    /// Query a workflow and return the lossless fixed Avro Value result.
2926    pub async fn query_workflow_avro_value<T: Serialize>(
2927        &self,
2928        workflow_id: &str,
2929        query_name: &str,
2930        input: T,
2931    ) -> Result<AvroValue> {
2932        self.query_workflow_avro_value_target(workflow_id, None, query_name, input)
2933            .await
2934    }
2935
2936    /// Query a selected run and return the lossless fixed Avro Value result.
2937    pub async fn query_workflow_run_avro_value<T: Serialize>(
2938        &self,
2939        workflow_id: &str,
2940        run_id: &str,
2941        query_name: &str,
2942        input: T,
2943    ) -> Result<AvroValue> {
2944        self.query_workflow_avro_value_target(workflow_id, Some(run_id), query_name, input)
2945            .await
2946    }
2947
2948    async fn query_workflow_avro_value_target<T: Serialize>(
2949        &self,
2950        workflow_id: &str,
2951        run_id: Option<&str>,
2952        query_name: &str,
2953        input: T,
2954    ) -> Result<AvroValue> {
2955        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2956        let body = json!({"input": encode_typed_envelope(&input, DEFAULT_CODEC)?});
2957        let path = match run_id {
2958            Some(run_id) => {
2959                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
2960            }
2961            None => format!("/workflows/{workflow_id}/query/{query_name}"),
2962        };
2963        let response: Value = match self
2964            .request_json(
2965                reqwest::Method::POST,
2966                &path,
2967                RequestProtocol::ControlPlane,
2968                Some(&body),
2969            )
2970            .await
2971        {
2972            Ok(response) => response,
2973            Err(Error::Http { status, body }) => {
2974                return Err(Error::QueryFailed(query_failure(status, body)));
2975            }
2976            Err(error) => return Err(error),
2977        };
2978
2979        let envelope = response
2980            .get("result_envelope")
2981            .filter(|envelope| !envelope.is_null())
2982            .ok_or_else(|| {
2983                Error::Codec(
2984                    "missing_payload_envelope: typed query result requires result_envelope"
2985                        .to_string(),
2986                )
2987            })?;
2988        decode_wire_avro_value(envelope, DEFAULT_CODEC)
2989    }
2990
2991    async fn query_workflow_target<T: Serialize>(
2992        &self,
2993        workflow_id: &str,
2994        run_id: Option<&str>,
2995        query_name: &str,
2996        input: T,
2997    ) -> Result<Value> {
2998        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
2999        let input_envelope = encode_typed_envelope(&input, DEFAULT_CODEC)?;
3000        let body = json!({
3001            "input": input_envelope
3002        });
3003        let path = match run_id {
3004            Some(run_id) => {
3005                format!("/workflows/{workflow_id}/runs/{run_id}/query/{query_name}")
3006            }
3007            None => format!("/workflows/{workflow_id}/query/{query_name}"),
3008        };
3009        let response: Value = match self
3010            .request_json(
3011                reqwest::Method::POST,
3012                &path,
3013                RequestProtocol::ControlPlane,
3014                Some(&body),
3015            )
3016            .await
3017        {
3018            Ok(response) => response,
3019            Err(Error::Http { status, body }) => {
3020                return Err(Error::QueryFailed(query_failure(status, body)));
3021            }
3022            Err(error) => return Err(error),
3023        };
3024
3025        if let Some(envelope) = response
3026            .get("result_envelope")
3027            .filter(|envelope| !envelope.is_null())
3028        {
3029            return decode_wire_value(envelope, DEFAULT_CODEC);
3030        }
3031
3032        Ok(response.get("result").cloned().unwrap_or(Value::Null))
3033    }
3034
3035    /// Send a synchronous update using fixed Avro Value arguments.
3036    pub async fn update_workflow<T: Serialize>(
3037        &self,
3038        workflow_id: &str,
3039        update_name: &str,
3040        input: T,
3041        request_id: Option<&str>,
3042    ) -> Result<Value> {
3043        let response = self
3044            .update_workflow_response(workflow_id, update_name, input, request_id)
3045            .await?;
3046        if let Some(envelope) = response
3047            .get("result_envelope")
3048            .filter(|envelope| !envelope.is_null())
3049        {
3050            return decode_wire_value(envelope, DEFAULT_CODEC);
3051        }
3052        Ok(response.get("result").cloned().unwrap_or(response))
3053    }
3054
3055    /// Send a synchronous update and retain a bytes-capable Avro result.
3056    pub async fn update_workflow_avro_value<T: Serialize>(
3057        &self,
3058        workflow_id: &str,
3059        update_name: &str,
3060        input: T,
3061        request_id: Option<&str>,
3062    ) -> Result<AvroValue> {
3063        let response = self
3064            .update_workflow_response(workflow_id, update_name, input, request_id)
3065            .await?;
3066        let envelope = response
3067            .get("result_envelope")
3068            .filter(|envelope| !envelope.is_null())
3069            .ok_or_else(|| {
3070                Error::Codec(
3071                    "missing_payload_envelope: typed update result requires result_envelope"
3072                        .to_string(),
3073                )
3074            })?;
3075        decode_wire_avro_value(envelope, DEFAULT_CODEC)
3076    }
3077
3078    async fn update_workflow_response<T: Serialize>(
3079        &self,
3080        workflow_id: &str,
3081        update_name: &str,
3082        input: T,
3083        request_id: Option<&str>,
3084    ) -> Result<Value> {
3085        let input = normalize_avro_arguments(AvroValue::from_serialize(&input)?);
3086        let mut body = json!({
3087            "input": encode_typed_envelope(&input, DEFAULT_CODEC)?,
3088            "wait_for": "completed",
3089        });
3090        if let Some(request_id) = request_id {
3091            body["request_id"] = json!(request_id);
3092        }
3093        self.request_json(
3094            reqwest::Method::POST,
3095            &format!("/workflows/{workflow_id}/update/{update_name}"),
3096            RequestProtocol::ControlPlane,
3097            Some(&body),
3098        )
3099        .await
3100    }
3101
3102    pub async fn describe_workflow(&self, workflow_id: &str) -> Result<WorkflowDescription> {
3103        let path = format!("/workflows/{workflow_id}");
3104        let mut data: WorkflowDescription = self
3105            .request_json(
3106                reqwest::Method::GET,
3107                &path,
3108                RequestProtocol::ControlPlane,
3109                Option::<&Value>::None,
3110            )
3111            .await?;
3112        data.decode_payloads()?;
3113        Ok(data)
3114    }
3115
3116    /// Describe one selected run, including historical terminal runs.
3117    pub async fn describe_workflow_run(
3118        &self,
3119        workflow_id: &str,
3120        run_id: &str,
3121    ) -> Result<WorkflowDescription> {
3122        let path = format!("/workflows/{workflow_id}/runs/{run_id}");
3123        let mut data: WorkflowDescription = self
3124            .request_json(
3125                reqwest::Method::GET,
3126                &path,
3127                RequestProtocol::ControlPlane,
3128                Option::<&Value>::None,
3129            )
3130            .await?;
3131        data.decode_payloads()?;
3132        Ok(data)
3133    }
3134
3135    fn workflow_stream_path(workflow_id: &str, run_id: &str, stream_name: Option<&str>) -> String {
3136        let mut path = format!(
3137            "/workflows/{}/runs/{}/streams",
3138            percent_encode_path_segment(workflow_id),
3139            percent_encode_path_segment(run_id),
3140        );
3141        if let Some(stream_name) = stream_name {
3142            path.push('/');
3143            path.push_str(&percent_encode_path_segment(stream_name));
3144        }
3145        path
3146    }
3147
3148    /// List the run-scoped output streams already opened by a workflow.
3149    pub async fn list_workflow_streams(
3150        &self,
3151        workflow_id: &str,
3152        run_id: &str,
3153    ) -> Result<Vec<WorkflowStreamDescription>> {
3154        let response: WorkflowStreamListResponse = self
3155            .request_json(
3156                reqwest::Method::GET,
3157                &Self::workflow_stream_path(workflow_id, run_id, None),
3158                RequestProtocol::ControlPlane,
3159                Option::<&Value>::None,
3160            )
3161            .await?;
3162        Ok(response.streams)
3163    }
3164
3165    /// Describe stream lifecycle, offsets, pending count, and terminal error.
3166    pub async fn describe_workflow_stream(
3167        &self,
3168        workflow_id: &str,
3169        run_id: &str,
3170        stream_name: &str,
3171    ) -> Result<WorkflowStreamDescription> {
3172        let response: WorkflowStreamDescriptionResponse = self
3173            .request_json(
3174                reqwest::Method::GET,
3175                &Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3176                RequestProtocol::ControlPlane,
3177                Option::<&Value>::None,
3178            )
3179            .await?;
3180        Ok(response.stream)
3181    }
3182
3183    /// Read one bounded page beginning at a zero-based offset.
3184    ///
3185    /// Delivery is at least once: persist `next_offset` only after processing
3186    /// the page. The future is cancellation-safe; dropping it cancels the
3187    /// in-flight request. Long polling is capped at 60 seconds by the SDK and
3188    /// service contract.
3189    pub async fn subscribe_workflow_stream(
3190        &self,
3191        workflow_id: &str,
3192        run_id: &str,
3193        stream_name: &str,
3194        from_offset: u64,
3195        max_items: usize,
3196        wait: Duration,
3197    ) -> Result<WorkflowStreamPage> {
3198        let max_items = max_items.clamp(1, 500);
3199        let wait_seconds = wait.as_secs().min(MAX_LONG_POLL_TIMEOUT_SECONDS);
3200        let path = format!(
3201            "{}/items?from={from_offset}&max_items={max_items}&wait_seconds={wait_seconds}",
3202            Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3203        );
3204        let response: WorkflowStreamPageResponse = self
3205            .request_json_with_timeout(
3206                reqwest::Method::GET,
3207                &path,
3208                RequestProtocol::ControlPlane,
3209                Option::<&Value>::None,
3210                Duration::from_secs(wait_seconds.saturating_add(5).max(5)),
3211            )
3212            .await?;
3213
3214        let items = response
3215            .items
3216            .into_iter()
3217            .map(|raw| {
3218                let offset = raw.get("offset").and_then(Value::as_u64).unwrap_or(0);
3219                let envelope = raw.get("payload").cloned();
3220                let payload = envelope
3221                    .as_ref()
3222                    .filter(|value| value.get("blob").is_some())
3223                    .map(|value| decode_wire_avro_value(value, DEFAULT_CODEC))
3224                    .transpose()?
3225                    .map(AvroValue::into_json)
3226                    .transpose()?;
3227                Ok(WorkflowStreamItem {
3228                    offset,
3229                    payload,
3230                    payload_envelope: envelope,
3231                    payload_reference: raw
3232                        .get("payload_reference")
3233                        .and_then(Value::as_str)
3234                        .map(str::to_string),
3235                    payload_codec: raw
3236                        .get("payload_codec")
3237                        .and_then(Value::as_str)
3238                        .map(str::to_string),
3239                    idempotency_key: raw
3240                        .get("idempotency_key")
3241                        .and_then(Value::as_str)
3242                        .map(str::to_string),
3243                    item_type: raw
3244                        .get("item_type")
3245                        .and_then(Value::as_str)
3246                        .map(str::to_string),
3247                    content_type: raw
3248                        .get("content_type")
3249                        .and_then(Value::as_str)
3250                        .map(str::to_string),
3251                    origin: raw
3252                        .get("origin")
3253                        .and_then(Value::as_str)
3254                        .map(str::to_string),
3255                    origin_reference: raw
3256                        .get("origin_reference")
3257                        .and_then(Value::as_str)
3258                        .map(str::to_string),
3259                    emitted_at: raw
3260                        .get("emitted_at")
3261                        .and_then(Value::as_str)
3262                        .map(str::to_string),
3263                    raw,
3264                })
3265            })
3266            .collect::<Result<Vec<_>>>()?;
3267        Ok(WorkflowStreamPage {
3268            stream: response.stream,
3269            items,
3270            next_offset: response.next_offset,
3271            terminal: response.terminal,
3272        })
3273    }
3274
3275    /// Append inline Avro envelopes or opaque external payload references.
3276    pub async fn append_workflow_stream(
3277        &self,
3278        workflow_id: &str,
3279        run_id: &str,
3280        stream_name: &str,
3281        items: &[WorkflowStreamAppendItem],
3282        max_pending_items: Option<u64>,
3283    ) -> Result<WorkflowStreamAppendResult> {
3284        if items.is_empty() {
3285            return Err(Error::Codec(
3286                "workflow_stream_items_empty: append requires at least one item".to_string(),
3287            ));
3288        }
3289        let mut body = json!({
3290            "items": items
3291                .iter()
3292                .map(|item| item.wire_value(None))
3293                .collect::<Vec<_>>(),
3294        });
3295        if let Some(max_pending_items) = max_pending_items {
3296            if max_pending_items == 0 {
3297                return Err(Error::Codec(
3298                    "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
3299                        .to_string(),
3300                ));
3301            }
3302            body["max_pending_items"] = json!(max_pending_items);
3303        }
3304        let response: WorkflowStreamAppendResponse = self
3305            .request_json(
3306                reqwest::Method::POST,
3307                &format!(
3308                    "{}/items",
3309                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3310                ),
3311                RequestProtocol::ControlPlane,
3312                Some(&body),
3313            )
3314            .await?;
3315        Ok(WorkflowStreamAppendResult {
3316            stream: response.stream,
3317            accepted_offsets: response.accepted_offsets,
3318            accepted: response.accepted,
3319            deduped: response.deduped,
3320        })
3321    }
3322
3323    /// Close a stream, or mark it errored when `error_reason` is supplied.
3324    pub async fn close_workflow_stream(
3325        &self,
3326        workflow_id: &str,
3327        run_id: &str,
3328        stream_name: &str,
3329        error_reason: Option<&str>,
3330        retention_seconds: Option<u64>,
3331    ) -> Result<WorkflowStreamDescription> {
3332        let mut body = json!({});
3333        if let Some(error_reason) = error_reason {
3334            body["error_reason"] = json!(error_reason);
3335        }
3336        if let Some(retention_seconds) = retention_seconds {
3337            if retention_seconds == 0 {
3338                return Err(Error::Codec(
3339                    "workflow_stream_retention_invalid: retention_seconds must be positive"
3340                        .to_string(),
3341                ));
3342            }
3343            body["retention_seconds"] = json!(retention_seconds);
3344        }
3345        let response: WorkflowStreamDescriptionResponse = self
3346            .request_json(
3347                reqwest::Method::POST,
3348                &format!(
3349                    "{}/close",
3350                    Self::workflow_stream_path(workflow_id, run_id, Some(stream_name)),
3351                ),
3352                RequestProtocol::ControlPlane,
3353                Some(&body),
3354            )
3355            .await?;
3356        Ok(response.stream)
3357    }
3358
3359    pub async fn register_worker(
3360        &self,
3361        worker_id: &str,
3362        task_queue: &str,
3363        supported_workflow_types: Vec<String>,
3364        supported_activity_types: Vec<String>,
3365        max_concurrent_workflow_tasks: usize,
3366        max_concurrent_activity_tasks: usize,
3367    ) -> Result<RegisterWorkerResponse> {
3368        self.register_worker_with_capabilities(
3369            worker_id,
3370            task_queue,
3371            supported_workflow_types,
3372            supported_activity_types,
3373            max_concurrent_workflow_tasks,
3374            max_concurrent_activity_tasks,
3375            Vec::new(),
3376        )
3377        .await
3378    }
3379
3380    /// Register a worker and explicitly advertise additive worker capabilities.
3381    pub async fn register_worker_with_capabilities(
3382        &self,
3383        worker_id: &str,
3384        task_queue: &str,
3385        supported_workflow_types: Vec<String>,
3386        supported_activity_types: Vec<String>,
3387        max_concurrent_workflow_tasks: usize,
3388        max_concurrent_activity_tasks: usize,
3389        capabilities: Vec<String>,
3390    ) -> Result<RegisterWorkerResponse> {
3391        self.register_worker_with_command_contracts(
3392            worker_id,
3393            task_queue,
3394            supported_workflow_types,
3395            supported_activity_types,
3396            max_concurrent_workflow_tasks,
3397            max_concurrent_activity_tasks,
3398            capabilities,
3399            Value::Object(serde_json::Map::new()),
3400        )
3401        .await
3402    }
3403
3404    /// Register a worker and advertise its named query and update handlers.
3405    ///
3406    /// This Rust SDK cannot execute synchronous pre-accept update validation,
3407    /// so a workflow contract with a non-empty or malformed
3408    /// `update_validators` declaration returns
3409    /// [`Error::UnsupportedUpdateValidators`] before registration transport.
3410    #[allow(clippy::too_many_arguments)]
3411    pub async fn register_worker_with_command_contracts(
3412        &self,
3413        worker_id: &str,
3414        task_queue: &str,
3415        supported_workflow_types: Vec<String>,
3416        supported_activity_types: Vec<String>,
3417        max_concurrent_workflow_tasks: usize,
3418        max_concurrent_activity_tasks: usize,
3419        capabilities: Vec<String>,
3420        workflow_command_contracts: Value,
3421    ) -> Result<RegisterWorkerResponse> {
3422        if let Some(contracts) = workflow_command_contracts.as_object() {
3423            for (workflow_type, contract) in contracts {
3424                let Some(update_validators) = contract.get("update_validators") else {
3425                    continue;
3426                };
3427                if !update_validators
3428                    .as_array()
3429                    .is_some_and(|validators| validators.is_empty())
3430                {
3431                    return Err(Error::UnsupportedUpdateValidators {
3432                        workflow_type: workflow_type.clone(),
3433                    });
3434                }
3435            }
3436        }
3437
3438        let mut body = json!({
3439            "worker_id": worker_id,
3440            "task_queue": task_queue,
3441            "runtime": "rust",
3442            "sdk_version": SDK_VERSION,
3443            "supported_workflow_types": supported_workflow_types,
3444            "supported_activity_types": supported_activity_types,
3445            "capabilities": capabilities,
3446            "capability_manifest": portable_worker_affinity_capability_manifest(),
3447            "max_concurrent_workflow_tasks": max_concurrent_workflow_tasks,
3448            "max_concurrent_activity_tasks": max_concurrent_activity_tasks
3449        });
3450        if workflow_command_contracts
3451            .as_object()
3452            .is_some_and(|contracts| !contracts.is_empty())
3453        {
3454            body["workflow_command_contracts"] = workflow_command_contracts;
3455        }
3456
3457        self.request_json(
3458            reqwest::Method::POST,
3459            "/worker/register",
3460            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3461            Some(&body),
3462        )
3463        .await
3464    }
3465
3466    /// Gracefully remove one worker's registration through the worker plane.
3467    ///
3468    /// This operation is separate from operator-facing worker management. It
3469    /// uses worker-protocol authentication and returns the server's lease
3470    /// recovery result.
3471    pub async fn deregister_worker_registration(
3472        &self,
3473        worker_id: &str,
3474    ) -> Result<WorkerDeregistrationEnvelope> {
3475        let path = format!(
3476            "/worker/registrations/{}",
3477            percent_encode_path_segment(worker_id)
3478        );
3479        self.request_json(
3480            reqwest::Method::DELETE,
3481            &path,
3482            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3483            Option::<&Value>::None,
3484        )
3485        .await
3486    }
3487
3488    /// Long-poll for an ephemeral, read-only workflow query task.
3489    pub async fn poll_query_task(
3490        &self,
3491        worker_id: &str,
3492        task_queue: &str,
3493        timeout: Duration,
3494    ) -> Result<Option<QueryTask>> {
3495        Ok(self
3496            .poll_query_task_response(worker_id, task_queue, timeout)
3497            .await?
3498            .task)
3499    }
3500
3501    /// Poll a query task while preserving server stop and drain metadata.
3502    pub async fn poll_query_task_response(
3503        &self,
3504        worker_id: &str,
3505        task_queue: &str,
3506        timeout: Duration,
3507    ) -> Result<PollQueryTaskResponse> {
3508        let poll_request_id = unique_request_id("rust-query-poll");
3509        self.poll_query_task_response_with_request_id(
3510            worker_id,
3511            task_queue,
3512            timeout,
3513            &poll_request_id,
3514            1,
3515        )
3516        .await
3517    }
3518
3519    async fn poll_query_task_response_with_request_id(
3520        &self,
3521        worker_id: &str,
3522        task_queue: &str,
3523        timeout: Duration,
3524        poll_request_id: &str,
3525        transport_retries: usize,
3526    ) -> Result<PollQueryTaskResponse> {
3527        let timeout_seconds = long_poll_timeout_seconds(timeout);
3528        let body = json!({
3529            "worker_id": worker_id,
3530            "task_queue": task_queue,
3531            "poll_request_id": poll_request_id,
3532            "timeout_seconds": timeout_seconds,
3533        });
3534        self.poll_request_json(
3535            "/worker/query-tasks/poll",
3536            RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3537            &body,
3538            timeout + Duration::from_secs(5),
3539            transport_retries,
3540        )
3541        .await
3542    }
3543
3544    /// Complete a query task without appending workflow history.
3545    pub async fn complete_query_task<T: Serialize>(
3546        &self,
3547        query_task_id: &str,
3548        lease_owner: &str,
3549        query_task_attempt: u64,
3550        result: T,
3551        codec: &str,
3552    ) -> Result<Value> {
3553        let typed_result = AvroValue::from_serialize(&result)?;
3554        let result_envelope = encode_typed_envelope(&typed_result, codec)?;
3555        self.complete_query_task_with_envelope(
3556            query_task_id,
3557            lease_owner,
3558            query_task_attempt,
3559            typed_result.into_json()?,
3560            result_envelope,
3561        )
3562        .await
3563    }
3564
3565    async fn complete_query_task_with_envelope(
3566        &self,
3567        query_task_id: &str,
3568        lease_owner: &str,
3569        query_task_attempt: u64,
3570        result: Value,
3571        result_envelope: Value,
3572    ) -> Result<Value> {
3573        let body = json!({
3574            "lease_owner": lease_owner,
3575            "query_task_attempt": query_task_attempt,
3576            "result": result,
3577            "result_envelope": result_envelope,
3578        });
3579        let path = format!("/worker/query-tasks/{query_task_id}/complete");
3580        let response = self
3581            .request_json(
3582                reqwest::Method::POST,
3583                &path,
3584                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3585                Some(&body),
3586            )
3587            .await;
3588        query_task_response(response)
3589    }
3590
3591    /// Report a stable machine-readable query-task failure.
3592    pub async fn fail_query_task(
3593        &self,
3594        query_task_id: &str,
3595        lease_owner: &str,
3596        query_task_attempt: u64,
3597        message: impl Into<String>,
3598        reason: impl Into<String>,
3599        failure_type: impl Into<String>,
3600    ) -> Result<Value> {
3601        let body = json!({
3602            "lease_owner": lease_owner,
3603            "query_task_attempt": query_task_attempt,
3604            "failure": {
3605                "message": message.into(),
3606                "reason": reason.into(),
3607                "type": failure_type.into(),
3608            }
3609        });
3610        let path = format!("/worker/query-tasks/{query_task_id}/fail");
3611        let response = self
3612            .request_json(
3613                reqwest::Method::POST,
3614                &path,
3615                RequestProtocol::Worker(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
3616                Some(&body),
3617            )
3618            .await;
3619        query_task_response(response)
3620    }
3621
3622    pub async fn heartbeat_worker(
3623        &self,
3624        worker_id: &str,
3625        workflow_available: usize,
3626        activity_available: usize,
3627    ) -> Result<Value> {
3628        let body = json!({
3629            "worker_id": worker_id,
3630            "task_slots": {
3631                "workflow_available": workflow_available,
3632                "activity_available": activity_available
3633            },
3634            "process_metrics": {
3635                "process_id": std::process::id(),
3636                "process_uptime_seconds": 0
3637            }
3638        });
3639
3640        self.request_json(
3641            reqwest::Method::POST,
3642            "/worker/heartbeat",
3643            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3644            Some(&body),
3645        )
3646        .await
3647    }
3648
3649    pub async fn poll_workflow_task(
3650        &self,
3651        worker_id: &str,
3652        task_queue: &str,
3653        timeout: Duration,
3654    ) -> Result<Option<WorkflowTask>> {
3655        Ok(self
3656            .poll_workflow_task_response(worker_id, task_queue, timeout)
3657            .await?
3658            .task)
3659    }
3660
3661    pub async fn poll_workflow_task_response(
3662        &self,
3663        worker_id: &str,
3664        task_queue: &str,
3665        timeout: Duration,
3666    ) -> Result<PollWorkflowTaskResponse> {
3667        let poll_request_id = unique_request_id("rust-workflow-poll");
3668        self.poll_workflow_task_response_with_request_id(
3669            worker_id,
3670            task_queue,
3671            timeout,
3672            &poll_request_id,
3673            1,
3674        )
3675        .await
3676    }
3677
3678    async fn poll_workflow_task_response_with_request_id(
3679        &self,
3680        worker_id: &str,
3681        task_queue: &str,
3682        timeout: Duration,
3683        poll_request_id: &str,
3684        transport_retries: usize,
3685    ) -> Result<PollWorkflowTaskResponse> {
3686        let body = json!({
3687            "worker_id": worker_id,
3688            "task_queue": task_queue,
3689            "poll_request_id": poll_request_id,
3690            "timeout_seconds": long_poll_timeout_seconds(timeout),
3691        });
3692        let mut data: PollWorkflowTaskResponse = self
3693            .poll_request_json(
3694                "/worker/workflow-tasks/poll",
3695                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3696                &body,
3697                timeout + Duration::from_secs(5),
3698                transport_retries,
3699            )
3700            .await?;
3701
3702        if let Some(task) = data.task.as_mut() {
3703            self.fetch_remaining_workflow_history(worker_id, task)
3704                .await?;
3705        }
3706
3707        Ok(data)
3708    }
3709
3710    async fn fetch_remaining_workflow_history(
3711        &self,
3712        worker_id: &str,
3713        task: &mut WorkflowTask,
3714    ) -> Result<()> {
3715        let mut next_token = task.next_history_page_token.clone();
3716
3717        while let Some(token) = next_token.take().filter(|token| !token.is_empty()) {
3718            let lease_owner = task
3719                .lease_owner
3720                .clone()
3721                .unwrap_or_else(|| worker_id.to_string());
3722            let page = self
3723                .workflow_task_history_page(
3724                    &task.task_id,
3725                    &lease_owner,
3726                    task.workflow_task_attempt,
3727                    &token,
3728                )
3729                .await?;
3730
3731            task.append_history_page(page);
3732
3733            if task.next_history_page_token.as_deref() == Some(token.as_str()) {
3734                return Err(Error::Codec(
3735                    "workflow history pagination returned the same page token".to_string(),
3736                ));
3737            }
3738
3739            next_token = task.next_history_page_token.clone();
3740        }
3741
3742        Ok(())
3743    }
3744
3745    async fn workflow_task_history_page(
3746        &self,
3747        task_id: &str,
3748        lease_owner: &str,
3749        workflow_task_attempt: u64,
3750        next_history_page_token: &str,
3751    ) -> Result<WorkflowTaskHistoryPage> {
3752        let body = json!({
3753            "lease_owner": lease_owner,
3754            "workflow_task_attempt": workflow_task_attempt,
3755            "next_history_page_token": next_history_page_token
3756        });
3757        let path = format!("/worker/workflow-tasks/{task_id}/history");
3758
3759        self.request_json(
3760            reqwest::Method::POST,
3761            &path,
3762            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3763            Some(&body),
3764        )
3765        .await
3766    }
3767
3768    pub async fn complete_workflow_task(
3769        &self,
3770        task_id: &str,
3771        lease_owner: &str,
3772        workflow_task_attempt: u64,
3773        commands: Vec<Value>,
3774    ) -> Result<Value> {
3775        self.complete_workflow_task_with_message_streams(
3776            task_id,
3777            lease_owner,
3778            workflow_task_attempt,
3779            commands,
3780            Vec::new(),
3781            Vec::new(),
3782        )
3783        .await
3784    }
3785
3786    async fn complete_workflow_task_with_message_streams(
3787        &self,
3788        task_id: &str,
3789        lease_owner: &str,
3790        workflow_task_attempt: u64,
3791        commands: Vec<Value>,
3792        message_stream_cursors: Vec<Value>,
3793        message_stream_waits: Vec<Value>,
3794    ) -> Result<Value> {
3795        validate_workflow_task_commands(&commands)?;
3796        let has_message_stream_metadata =
3797            !message_stream_cursors.is_empty() || !message_stream_waits.is_empty();
3798        if has_message_stream_metadata
3799            && !worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
3800        {
3801            return Err(Error::Codec(
3802                "message_streams_unavailable: message stream completion metadata requires worker protocol 1.15 or newer"
3803                    .to_string(),
3804            ));
3805        }
3806        let protocol_version = workflow_completion_protocol_version_with_message_streams(
3807            &commands,
3808            has_message_stream_metadata,
3809        );
3810        let mut body = json!({
3811            "lease_owner": lease_owner,
3812            "workflow_task_attempt": workflow_task_attempt,
3813            "commands": commands
3814        });
3815        if !message_stream_cursors.is_empty() {
3816            body["message_stream_cursors"] = Value::Array(message_stream_cursors);
3817        }
3818        if !message_stream_waits.is_empty() {
3819            body["message_stream_waits"] = Value::Array(message_stream_waits);
3820        }
3821        let path = format!("/worker/workflow-tasks/{task_id}/complete");
3822        self.request_json(
3823            reqwest::Method::POST,
3824            &path,
3825            RequestProtocol::Worker(protocol_version),
3826            Some(&body),
3827        )
3828        .await
3829    }
3830
3831    pub async fn fail_workflow_task(
3832        &self,
3833        task_id: &str,
3834        lease_owner: &str,
3835        workflow_task_attempt: u64,
3836        message: impl Into<String>,
3837    ) -> Result<Value> {
3838        self.fail_workflow_task_with_type(
3839            task_id,
3840            lease_owner,
3841            workflow_task_attempt,
3842            message,
3843            "RustWorkflowTaskFailure",
3844        )
3845        .await
3846    }
3847
3848    async fn fail_workflow_task_with_type(
3849        &self,
3850        task_id: &str,
3851        lease_owner: &str,
3852        workflow_task_attempt: u64,
3853        message: impl Into<String>,
3854        failure_type: &str,
3855    ) -> Result<Value> {
3856        let body = json!({
3857            "lease_owner": lease_owner,
3858            "workflow_task_attempt": workflow_task_attempt,
3859            "failure": {
3860                "message": message.into(),
3861                "type": failure_type
3862            }
3863        });
3864        let path = format!("/worker/workflow-tasks/{task_id}/fail");
3865        self.request_json(
3866            reqwest::Method::POST,
3867            &path,
3868            RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3869            Some(&body),
3870        )
3871        .await
3872    }
3873
3874    pub async fn poll_activity_task(
3875        &self,
3876        worker_id: &str,
3877        task_queue: &str,
3878        timeout: Duration,
3879    ) -> Result<Option<ActivityTask>> {
3880        Ok(self
3881            .poll_activity_task_response(worker_id, task_queue, timeout)
3882            .await?
3883            .task)
3884    }
3885
3886    /// Poll an activity task while preserving server stop and drain metadata.
3887    pub async fn poll_activity_task_response(
3888        &self,
3889        worker_id: &str,
3890        task_queue: &str,
3891        timeout: Duration,
3892    ) -> Result<PollActivityTaskResponse> {
3893        let poll_request_id = unique_request_id("rust-activity-poll");
3894        self.poll_activity_task_response_with_request_id(
3895            worker_id,
3896            task_queue,
3897            timeout,
3898            &poll_request_id,
3899            1,
3900        )
3901        .await
3902    }
3903
3904    async fn poll_activity_task_response_with_request_id(
3905        &self,
3906        worker_id: &str,
3907        task_queue: &str,
3908        timeout: Duration,
3909        poll_request_id: &str,
3910        transport_retries: usize,
3911    ) -> Result<PollActivityTaskResponse> {
3912        let body = json!({
3913            "worker_id": worker_id,
3914            "task_queue": task_queue,
3915            "poll_request_id": poll_request_id,
3916            "timeout_seconds": long_poll_timeout_seconds(timeout),
3917        });
3918        let data: PollActivityTaskResponse = self
3919            .poll_request_json(
3920                "/worker/activity-tasks/poll",
3921                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3922                &body,
3923                timeout + Duration::from_secs(5),
3924                transport_retries,
3925            )
3926            .await?;
3927        Ok(data)
3928    }
3929
3930    pub async fn complete_activity_task<T: Serialize>(
3931        &self,
3932        task_id: &str,
3933        activity_attempt_id: &str,
3934        lease_owner: &str,
3935        result: T,
3936        codec: &str,
3937    ) -> Result<Value> {
3938        let result = encode_typed_envelope(&AvroValue::from_serialize(&result)?, codec)?;
3939        let body = json!({
3940            "activity_attempt_id": activity_attempt_id,
3941            "lease_owner": lease_owner,
3942            "result": result
3943        });
3944        let path = format!("/worker/activity-tasks/{task_id}/complete");
3945        activity_task_response(
3946            self.request_json(
3947                reqwest::Method::POST,
3948                &path,
3949                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3950                Some(&body),
3951            )
3952            .await,
3953            "complete",
3954            task_id,
3955            activity_attempt_id,
3956        )
3957    }
3958
3959    pub async fn fail_activity_task(
3960        &self,
3961        task_id: &str,
3962        activity_attempt_id: &str,
3963        lease_owner: &str,
3964        message: impl Into<String>,
3965        non_retryable: bool,
3966    ) -> Result<Value> {
3967        let body = json!({
3968            "activity_attempt_id": activity_attempt_id,
3969            "lease_owner": lease_owner,
3970            "failure": {
3971                "message": message.into(),
3972                "type": "RustActivityFailure",
3973                "non_retryable": non_retryable
3974            }
3975        });
3976        let path = format!("/worker/activity-tasks/{task_id}/fail");
3977        activity_task_response(
3978            self.request_json(
3979                reqwest::Method::POST,
3980                &path,
3981                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
3982                Some(&body),
3983            )
3984            .await,
3985            "fail",
3986            task_id,
3987            activity_attempt_id,
3988        )
3989    }
3990
3991    pub async fn heartbeat_activity_task<T: Serialize>(
3992        &self,
3993        task_id: &str,
3994        activity_attempt_id: &str,
3995        lease_owner: &str,
3996        details: T,
3997    ) -> Result<ActivityHeartbeatResponse> {
3998        let details = encode_typed_envelope(&AvroValue::from_serialize(&details)?, DEFAULT_CODEC)?;
3999        let body = json!({
4000            "activity_attempt_id": activity_attempt_id,
4001            "lease_owner": lease_owner,
4002            "details": details
4003        });
4004        let path = format!("/worker/activity-tasks/{task_id}/heartbeat");
4005        activity_task_response(
4006            self.request_json(
4007                reqwest::Method::POST,
4008                &path,
4009                RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
4010                Some(&body),
4011            )
4012            .await,
4013            "heartbeat",
4014            task_id,
4015            activity_attempt_id,
4016        )
4017    }
4018
4019    async fn request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4020        &self,
4021        method: reqwest::Method,
4022        path: &str,
4023        protocol: RequestProtocol,
4024        body: Option<&B>,
4025    ) -> Result<T> {
4026        self.request_json_with_timeout(method, path, protocol, body, Duration::from_secs(60))
4027            .await
4028    }
4029
4030    async fn request_json_with_timeout<T: DeserializeOwned, B: Serialize + ?Sized>(
4031        &self,
4032        method: reqwest::Method,
4033        path: &str,
4034        protocol: RequestProtocol,
4035        body: Option<&B>,
4036        timeout: Duration,
4037    ) -> Result<T> {
4038        let auth_token = self.auth_token(protocol)?;
4039        let mut request = self
4040            .http
4041            .request(method.clone(), format!("{}/api{}", self.base_url, path))
4042            .timeout(timeout)
4043            .header(reqwest::header::ACCEPT, "application/json")
4044            .header(reqwest::header::CONTENT_TYPE, "application/json")
4045            .header("X-Namespace", &self.namespace);
4046
4047        match protocol {
4048            RequestProtocol::Worker(version) => {
4049                request = request.header("X-Durable-Workflow-Protocol-Version", version);
4050            }
4051            RequestProtocol::ControlPlane => {
4052                request = request.header(
4053                    "X-Durable-Workflow-Control-Plane-Version",
4054                    CONTROL_PLANE_VERSION,
4055                );
4056            }
4057        }
4058
4059        if let Some(token) = auth_token {
4060            request = request.bearer_auth(token);
4061        }
4062
4063        if let Some(body) = body {
4064            let mut body = serde_json::to_value(body)?;
4065            if matches!(
4066                method,
4067                reqwest::Method::POST | reqwest::Method::PUT | reqwest::Method::PATCH
4068            ) {
4069                self.externalize_runtime_payloads(&mut body, path, protocol)
4070                    .await?;
4071            }
4072            request = request.json(&body);
4073        }
4074
4075        let request = request.build()?;
4076        let poll_request_id = path.ends_with("/poll").then(|| {
4077            request
4078                .body()
4079                .and_then(reqwest::Body::as_bytes)
4080                .and_then(|body| serde_json::from_slice::<Value>(body).ok())
4081                .and_then(|body| body.get("poll_request_id")?.as_str().map(str::to_owned))
4082                .unwrap_or_default()
4083        });
4084        let mut storage_retries = 0_usize;
4085
4086        loop {
4087            // Keep the serialized body and lease/poll identity across admission pauses.
4088            let response = self
4089                .http
4090                .execute(request.try_clone().ok_or_else(|| {
4091                    Error::WorkerLoop("worker request body cannot be retried".to_string())
4092                })?)
4093                .await?;
4094            let status = response.status();
4095            let bytes = response.bytes().await?;
4096
4097            if !status.is_success() {
4098                let body = String::from_utf8_lossy(&bytes).to_string();
4099                if let Some(protocol) = protocol_failure(status, &body) {
4100                    return Err(Error::Protocol(protocol));
4101                }
4102                let error = Error::Http { status, body };
4103                if self
4104                    .wait_for_storage_admission(
4105                        &error,
4106                        protocol,
4107                        poll_request_id.as_deref(),
4108                        &mut storage_retries,
4109                    )
4110                    .await
4111                {
4112                    continue;
4113                }
4114                return Err(error);
4115            }
4116
4117            if bytes.is_empty() {
4118                return Ok(serde_json::from_value(Value::Null)?);
4119            }
4120
4121            let mut value: Value = serde_json::from_slice(&bytes)?;
4122            self.resolve_runtime_payloads(&mut value, path, protocol)
4123                .await?;
4124            return Ok(serde_json::from_value(value)?);
4125        }
4126    }
4127
4128    async fn wait_for_storage_admission(
4129        &self,
4130        error: &Error,
4131        protocol: RequestProtocol,
4132        poll_request_id: Option<&str>,
4133        retries: &mut usize,
4134    ) -> bool {
4135        let Some(admission) = self
4136            .worker_storage_admission
4137            .as_ref()
4138            .filter(|_| matches!(protocol, RequestProtocol::Worker(_)))
4139        else {
4140            return false;
4141        };
4142        let Some(advertised_delay) = worker_storage_admission_retry_after(error, poll_request_id)
4143        else {
4144            return false;
4145        };
4146        *retries = retries.saturating_add(1);
4147        let delay = worker_retry_delay(admission.policy, *retries)
4148            .max(advertised_delay)
4149            .min(admission.policy.max_backoff.max(Duration::from_millis(1)));
4150        let deadline = tokio::time::Instant::now() + delay;
4151        loop {
4152            if admission.stop.load(Ordering::SeqCst) {
4153                return false;
4154            }
4155            let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
4156            if remaining.is_zero() {
4157                return true;
4158            }
4159            tokio::time::sleep(remaining.min(Duration::from_millis(100))).await;
4160        }
4161    }
4162
4163    async fn poll_request_json<T: DeserializeOwned, B: Serialize + ?Sized>(
4164        &self,
4165        path: &str,
4166        protocol: RequestProtocol,
4167        body: &B,
4168        timeout: Duration,
4169        max_retries: usize,
4170    ) -> Result<T> {
4171        let mut retries = 0;
4172
4173        loop {
4174            let response = self
4175                .request_json_with_timeout(
4176                    reqwest::Method::POST,
4177                    path,
4178                    protocol,
4179                    Some(body),
4180                    timeout,
4181                )
4182                .await;
4183
4184            match response {
4185                Err(Error::Transport(_)) if retries < max_retries => retries += 1,
4186                response => return worker_poll_response(response),
4187            }
4188        }
4189    }
4190
4191    fn auth_token(&self, protocol: RequestProtocol) -> Result<Option<&str>> {
4192        match protocol {
4193            RequestProtocol::Worker(_) => {
4194                if let Some(token) = self.worker_token.as_deref().or(self.token.as_deref()) {
4195                    return Ok(Some(token));
4196                }
4197                if self.control_token.is_some() {
4198                    return Err(Error::MissingRoleCredentials {
4199                        role: "worker",
4200                        opposite_role: "control",
4201                    });
4202                }
4203                Ok(None)
4204            }
4205            RequestProtocol::ControlPlane => {
4206                if let Some(token) = self.control_token.as_deref().or(self.token.as_deref()) {
4207                    return Ok(Some(token));
4208                }
4209                if self.worker_token.is_some() {
4210                    return Err(Error::MissingRoleCredentials {
4211                        role: "control",
4212                        opposite_role: "worker",
4213                    });
4214                }
4215                Ok(None)
4216            }
4217        }
4218    }
4219}
4220
4221fn query_failure(status: reqwest::StatusCode, raw_body: String) -> QueryFailure {
4222    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4223    let reason = body
4224        .get("reason")
4225        .and_then(Value::as_str)
4226        .unwrap_or("query_rejected")
4227        .to_string();
4228    let message = body
4229        .get("message")
4230        .or_else(|| body.get("error"))
4231        .and_then(Value::as_str)
4232        .unwrap_or("workflow query was rejected")
4233        .to_string();
4234
4235    QueryFailure {
4236        status: status.as_u16(),
4237        reason,
4238        message,
4239        body,
4240    }
4241}
4242
4243fn workflow_command_result(
4244    command: WorkflowCommandKind,
4245    data: Value,
4246    workflow_id: &str,
4247    run_id: Option<&str>,
4248) -> WorkflowCommandResult {
4249    WorkflowCommandResult {
4250        command,
4251        workflow_id: data
4252            .get("workflow_id")
4253            .and_then(Value::as_str)
4254            .unwrap_or(workflow_id)
4255            .to_string(),
4256        run_id: data
4257            .get("run_id")
4258            .and_then(Value::as_str)
4259            .or(run_id)
4260            .map(str::to_string),
4261        outcome: data
4262            .get("outcome")
4263            .and_then(Value::as_str)
4264            .map(str::to_string),
4265        reason: data
4266            .get("reason")
4267            .and_then(Value::as_str)
4268            .map(str::to_string),
4269        command_status: data
4270            .get("command_status")
4271            .and_then(Value::as_str)
4272            .map(str::to_string),
4273        raw: data,
4274    }
4275}
4276
4277fn workflow_command_rejection(
4278    command: WorkflowCommandKind,
4279    status: reqwest::StatusCode,
4280    raw_body: String,
4281    workflow_id: &str,
4282    run_id: Option<&str>,
4283) -> WorkflowCommandRejection {
4284    let body = serde_json::from_str(&raw_body).unwrap_or_else(|_| json!({"message": raw_body}));
4285    WorkflowCommandRejection {
4286        command,
4287        status: status.as_u16(),
4288        reason: body
4289            .get("reason")
4290            .and_then(Value::as_str)
4291            .unwrap_or("workflow_command_rejected")
4292            .to_string(),
4293        message: body
4294            .get("message")
4295            .or_else(|| body.get("error"))
4296            .and_then(Value::as_str)
4297            .unwrap_or("workflow lifecycle command was rejected")
4298            .to_string(),
4299        workflow_id: body
4300            .get("workflow_id")
4301            .and_then(Value::as_str)
4302            .unwrap_or(workflow_id)
4303            .to_string(),
4304        run_id: body
4305            .get("run_id")
4306            .and_then(Value::as_str)
4307            .or(run_id)
4308            .map(str::to_string),
4309        target_scope: body
4310            .get("target_scope")
4311            .and_then(Value::as_str)
4312            .map(str::to_string),
4313        body,
4314    }
4315}
4316
4317fn query_task_response(response: Result<Value>) -> Result<Value> {
4318    match response {
4319        Err(Error::Http { status, body }) => Err(Error::QueryFailed(query_failure(status, body))),
4320        response => response,
4321    }
4322}
4323
4324fn worker_poll_response<T: DeserializeOwned>(response: Result<T>) -> Result<T> {
4325    match response {
4326        Err(Error::Http { status, body })
4327            if status == reqwest::StatusCode::CONFLICT && worker_poll_body_is_stop(&body) =>
4328        {
4329            Ok(serde_json::from_str(&body)?)
4330        }
4331        response => response,
4332    }
4333}
4334
4335fn worker_poll_body_is_stop(body: &str) -> bool {
4336    serde_json::from_str::<Value>(body)
4337        .ok()
4338        .is_some_and(|body| {
4339            worker_poll_is_stop(
4340                body.get("poll_status").and_then(Value::as_str),
4341                body.get("reason").and_then(Value::as_str),
4342            )
4343        })
4344}
4345
4346fn worker_poll_is_stop(poll_status: Option<&str>, reason: Option<&str>) -> bool {
4347    matches!(poll_status, Some("draining" | "stopped"))
4348        || matches!(reason, Some("worker_draining" | "worker_stopped"))
4349}
4350
4351fn query_task_rejection_is_final(error: &Error) -> bool {
4352    matches!(
4353        error,
4354        Error::QueryFailed(failure)
4355            if QUERY_TASK_FINAL_REJECTION_REASONS.contains(&failure.reason.as_str())
4356    )
4357}
4358
4359fn activity_task_response<T>(
4360    response: Result<T>,
4361    operation: &str,
4362    task_id: &str,
4363    activity_attempt_id: &str,
4364) -> Result<T> {
4365    match response {
4366        Err(Error::Http { status, body }) => {
4367            let body = serde_json::from_str(&body).unwrap_or_else(|_| json!({"message": body}));
4368            Err(Error::ActivityTaskRejected(ActivityTaskRejection {
4369                operation: operation.to_string(),
4370                status: status.as_u16(),
4371                reason: body
4372                    .get("reason")
4373                    .and_then(Value::as_str)
4374                    .unwrap_or("activity_task_rejected")
4375                    .to_string(),
4376                task_id: body
4377                    .get("task_id")
4378                    .and_then(Value::as_str)
4379                    .unwrap_or(task_id)
4380                    .to_string(),
4381                activity_attempt_id: body
4382                    .get("activity_attempt_id")
4383                    .and_then(Value::as_str)
4384                    .unwrap_or(activity_attempt_id)
4385                    .to_string(),
4386                cancel_requested: body
4387                    .get("cancel_requested")
4388                    .and_then(Value::as_bool)
4389                    .unwrap_or(false),
4390                can_continue: body.get("can_continue").and_then(Value::as_bool),
4391                run_closed_reason: body
4392                    .get("run_closed_reason")
4393                    .and_then(Value::as_str)
4394                    .map(str::to_string),
4395                body,
4396            }))
4397        }
4398        response => response,
4399    }
4400}
4401
4402fn activity_task_rejection_is_final(error: &Error) -> bool {
4403    matches!(
4404        error,
4405        Error::ActivityTaskRejected(rejection)
4406            if matches!(
4407                rejection.reason.as_str(),
4408                "run_cancelled"
4409                    | "run_terminated"
4410                    | "attempt_closed"
4411                    | "stale_attempt"
4412                    | "activity_cancelled"
4413                    | "task_cancelled"
4414                    | "run_closed"
4415                    | "activity_not_running"
4416                    | "attempt_not_found"
4417            )
4418    )
4419}
4420
4421fn workflow_task_completion_is_terminal_timeout(
4422    error: &Error,
4423    task_id: &str,
4424    workflow_task_attempt: u64,
4425    run_id: Option<&str>,
4426) -> bool {
4427    let Error::Http { status, body } = error else {
4428        return false;
4429    };
4430    if *status != reqwest::StatusCode::CONFLICT {
4431        return false;
4432    }
4433
4434    let Some(run_id) = run_id else {
4435        return false;
4436    };
4437    let Ok(body) = serde_json::from_str::<Value>(body) else {
4438        return false;
4439    };
4440
4441    body.get("recorded").and_then(Value::as_bool) == Some(false)
4442        && body.get("reason").and_then(Value::as_str) == Some("run_timed_out")
4443        && body.get("run_status").and_then(Value::as_str) == Some("failed")
4444        && body.get("run_id").and_then(Value::as_str) == Some(run_id)
4445        && body.get("task_id").and_then(Value::as_str) == Some(task_id)
4446        && body.get("workflow_task_attempt").and_then(Value::as_u64) == Some(workflow_task_attempt)
4447}
4448
4449fn protocol_failure(status: reqwest::StatusCode, raw_body: &str) -> Option<ProtocolFailure> {
4450    let body: Value = serde_json::from_str(raw_body).ok()?;
4451    let reason = body.get("reason")?.as_str()?;
4452    if !matches!(
4453        reason,
4454        "missing_protocol_version"
4455            | "unsupported_protocol_version"
4456            | "missing_control_plane_version"
4457            | "unsupported_control_plane_version"
4458    ) {
4459        return None;
4460    }
4461
4462    Some(ProtocolFailure {
4463        status: status.as_u16(),
4464        reason: reason.to_string(),
4465        message: body
4466            .get("message")
4467            .or_else(|| body.get("error"))
4468            .and_then(Value::as_str)
4469            .unwrap_or("protocol version rejected")
4470            .to_string(),
4471        supported_version: body
4472            .get("supported_version")
4473            .and_then(Value::as_str)
4474            .map(str::to_string),
4475        requested_version: body
4476            .get("requested_version")
4477            .and_then(Value::as_str)
4478            .map(str::to_string),
4479        body,
4480    })
4481}
4482
4483fn long_poll_timeout_seconds(timeout: Duration) -> u64 {
4484    timeout
4485        .as_secs()
4486        .saturating_add(u64::from(timeout.subsec_nanos() > 0))
4487        .min(MAX_LONG_POLL_TIMEOUT_SECONDS)
4488}
4489
4490fn worker_operation_is_retryable(error: &Error) -> bool {
4491    if worker_poll_capacity_retry_after(error).is_some()
4492        || worker_storage_admission_body(error).is_some()
4493        || worker_operation_is_explicitly_non_retryable(error)
4494    {
4495        return false;
4496    }
4497
4498    match error {
4499        Error::Transport(error) => {
4500            error.is_timeout() || error.is_connect() || error.is_request() || error.is_body()
4501        }
4502        Error::Http { status, .. } => {
4503            matches!(
4504                *status,
4505                reqwest::StatusCode::REQUEST_TIMEOUT | reqwest::StatusCode::TOO_MANY_REQUESTS
4506            ) || status.is_server_error()
4507        }
4508        _ => false,
4509    }
4510}
4511
4512fn worker_storage_admission_body(error: &Error) -> Option<Value> {
4513    let body: Value = match error {
4514        Error::Http { body, .. } => serde_json::from_str(body).ok()?,
4515        Error::ActivityTaskRejected(rejection) => rejection.body.clone(),
4516        _ => return None,
4517    };
4518    matches!(
4519        body.get("reason").and_then(Value::as_str),
4520        Some("storage_pressure" | "storage_admission_unavailable")
4521    )
4522    .then_some(body)
4523}
4524
4525fn worker_storage_admission_retry_after(
4526    error: &Error,
4527    poll_request_id: Option<&str>,
4528) -> Option<Duration> {
4529    let Error::Http { status, .. } = error else {
4530        return None;
4531    };
4532    let body = worker_storage_admission_body(error)?;
4533    let delay = body.get("retry_after_seconds")?.as_u64()?;
4534    if *status != reqwest::StatusCode::SERVICE_UNAVAILABLE
4535        || delay == 0
4536        || body.get("retryable") != Some(&Value::Bool(true))
4537        || !matches!(body.get("storage_state")?.as_str()?, "draining" | "fenced")
4538        || (body["reason"] == "storage_admission_unavailable" && body["storage_state"] != "fenced")
4539        || body
4540            .get("request_admitted")
4541            .is_some_and(|admitted| admitted != &Value::Bool(false))
4542    {
4543        return None;
4544    }
4545    match poll_request_id {
4546        Some(id) => {
4547            if id.is_empty()
4548                || body.get("task") != Some(&Value::Null)
4549                || body.get("poll_request_id").and_then(Value::as_str) != Some(id)
4550                || body.get("poll_status") != body.get("reason")
4551                || body.get("retry_same_poll_request_id") != Some(&Value::Bool(true))
4552                || body.get("claim_admitted") != Some(&Value::Bool(false))
4553            {
4554                return None;
4555            }
4556        }
4557        None if body.get("request_admitted") != Some(&Value::Bool(false)) => return None,
4558        None => {}
4559    }
4560    Some(Duration::from_secs(delay))
4561}
4562
4563fn worker_operation_is_explicitly_non_retryable(error: &Error) -> bool {
4564    let Error::Http { body, .. } = error else {
4565        return false;
4566    };
4567
4568    serde_json::from_str::<Value>(body)
4569        .ok()
4570        .and_then(|body| body.get("retryable").and_then(Value::as_bool))
4571        == Some(false)
4572}
4573
4574fn worker_poll_capacity_retry_after(error: &Error) -> Option<Duration> {
4575    let Error::Http { status, body } = error else {
4576        return None;
4577    };
4578    if *status != reqwest::StatusCode::TOO_MANY_REQUESTS {
4579        return None;
4580    }
4581
4582    let body = serde_json::from_str::<Value>(body).ok()?;
4583    let capacity_exhausted = body.get("poll_status").and_then(Value::as_str)
4584        == Some("long_poll_capacity_exhausted")
4585        || body.get("reason").and_then(Value::as_str) == Some("long_poll_capacity_exhausted");
4586    if !capacity_exhausted || body.get("retryable").and_then(Value::as_bool) != Some(true) {
4587        return None;
4588    }
4589
4590    Some(Duration::from_secs(
4591        body.get("retry_after_seconds")
4592            .and_then(Value::as_u64)
4593            .unwrap_or_default(),
4594    ))
4595}
4596
4597fn worker_retry_delay(policy: WorkerRetryPolicy, retry: usize) -> Duration {
4598    let exponent = retry.saturating_sub(1).min(31) as u32;
4599    policy
4600        .initial_backoff
4601        .saturating_mul(1_u32 << exponent)
4602        .min(policy.max_backoff)
4603}
4604
4605#[derive(Debug)]
4606pub struct ClientBuilder {
4607    base_url: String,
4608    token: Option<String>,
4609    control_token: Option<String>,
4610    worker_token: Option<String>,
4611    namespace: String,
4612    timeout: Duration,
4613    max_external_payload_bytes: usize,
4614}
4615
4616impl ClientBuilder {
4617    pub fn token(mut self, token: Option<String>) -> Self {
4618        self.token = token;
4619        self
4620    }
4621
4622    pub fn control_token(mut self, token: Option<String>) -> Self {
4623        self.control_token = token;
4624        self
4625    }
4626
4627    pub fn worker_token(mut self, token: Option<String>) -> Self {
4628        self.worker_token = token;
4629        self
4630    }
4631
4632    pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
4633        self.namespace = namespace.into();
4634        self
4635    }
4636
4637    pub fn timeout(mut self, timeout: Duration) -> Self {
4638        self.timeout = timeout;
4639        self
4640    }
4641
4642    /// Maximum unique external-payload bytes fetched per response (default 64 MiB).
4643    /// References are fetched only from this client's authenticated runtime.
4644    pub fn max_external_payload_bytes(mut self, bytes: usize) -> Self {
4645        self.max_external_payload_bytes = bytes;
4646        self
4647    }
4648
4649    pub fn build(self) -> Result<Client> {
4650        let base_url = self.base_url.trim_end_matches('/').to_string();
4651        let has_sdk_api_suffix = reqwest::Url::parse(&base_url)
4652            .map(|url| url.path().trim_end_matches('/').ends_with("/api"))
4653            .unwrap_or_else(|_| base_url.ends_with("/api"));
4654
4655        if has_sdk_api_suffix {
4656            return Err(Error::InvalidBaseUrl);
4657        }
4658
4659        Ok(Client {
4660            http: reqwest::Client::builder()
4661                .timeout(self.timeout)
4662                .redirect(reqwest::redirect::Policy::none())
4663                .build()?,
4664            base_url,
4665            token: self.token,
4666            control_token: self.control_token,
4667            worker_token: self.worker_token,
4668            namespace: self.namespace,
4669            max_external_payload_bytes: self.max_external_payload_bytes,
4670            worker_storage_admission: None,
4671            runtime_upload_policy: Arc::new(Mutex::new([None, None])),
4672        })
4673    }
4674}
4675
4676#[derive(Clone, Debug)]
4677pub struct WorkflowHandle {
4678    client: Client,
4679    pub workflow_id: String,
4680    pub run_id: Option<String>,
4681    pub workflow_type: String,
4682}
4683
4684impl WorkflowHandle {
4685    /// Describe whichever run is current for this stable workflow instance.
4686    pub async fn describe(&self) -> Result<WorkflowDescription> {
4687        self.client.describe_workflow(&self.workflow_id).await
4688    }
4689
4690    /// Describe the run identity originally selected by this handle.
4691    pub async fn describe_selected_run(&self) -> Result<WorkflowDescription> {
4692        let run_id = self.run_id.as_deref().ok_or_else(|| {
4693            Error::Codec("run_id is required for selected-run description".to_string())
4694        })?;
4695        self.client
4696            .describe_workflow_run(&self.workflow_id, run_id)
4697            .await
4698    }
4699
4700    pub async fn signal<T: Serialize>(&self, signal_name: &str, input: T) -> Result<Value> {
4701        self.client
4702            .signal_workflow(&self.workflow_id, signal_name, input)
4703            .await
4704    }
4705
4706    pub async fn append_message<T: Serialize>(
4707        &self,
4708        stream_name: &str,
4709        message_id: &str,
4710        input: T,
4711    ) -> Result<Value> {
4712        self.client
4713            .append_message_stream(&self.workflow_id, stream_name, message_id, input)
4714            .await
4715    }
4716
4717    /// Signal only if this handle's selected run is still current.
4718    pub async fn signal_selected_run<T: Serialize>(
4719        &self,
4720        signal_name: &str,
4721        input: T,
4722    ) -> Result<Value> {
4723        let run_id = self.run_id.as_deref().ok_or_else(|| {
4724            Error::Codec("run_id is required for selected-run signaling".to_string())
4725        })?;
4726        self.client
4727            .signal_workflow_run(&self.workflow_id, run_id, signal_name, input)
4728            .await
4729    }
4730
4731    /// Request cooperative cancellation of whichever run is current.
4732    pub async fn cancel(&self, options: WorkflowCommandOptions) -> Result<WorkflowCommandResult> {
4733        self.client
4734            .cancel_workflow(&self.workflow_id, options)
4735            .await
4736    }
4737
4738    /// Request cancellation only if this handle's selected run is still current.
4739    pub async fn cancel_selected_run(
4740        &self,
4741        options: WorkflowCommandOptions,
4742    ) -> Result<WorkflowCommandResult> {
4743        let run_id = self.run_id.as_deref().ok_or_else(|| {
4744            Error::Codec("run_id is required for selected-run cancellation".to_string())
4745        })?;
4746        self.client
4747            .cancel_workflow_run(&self.workflow_id, run_id, options)
4748            .await
4749    }
4750
4751    /// Forcefully terminate whichever run is current.
4752    pub async fn terminate(
4753        &self,
4754        options: WorkflowCommandOptions,
4755    ) -> Result<WorkflowCommandResult> {
4756        self.client
4757            .terminate_workflow(&self.workflow_id, options)
4758            .await
4759    }
4760
4761    /// Terminate only if this handle's selected run is still current.
4762    pub async fn terminate_selected_run(
4763        &self,
4764        options: WorkflowCommandOptions,
4765    ) -> Result<WorkflowCommandResult> {
4766        let run_id = self.run_id.as_deref().ok_or_else(|| {
4767            Error::Codec("run_id is required for selected-run termination".to_string())
4768        })?;
4769        self.client
4770            .terminate_workflow_run(&self.workflow_id, run_id, options)
4771            .await
4772    }
4773
4774    /// Execute a named, read-only query against this workflow.
4775    pub async fn query<T: Serialize>(&self, query_name: &str, input: T) -> Result<Value> {
4776        self.client
4777            .query_workflow(&self.workflow_id, query_name, input)
4778            .await
4779    }
4780
4781    pub async fn query_avro_value<T: Serialize>(
4782        &self,
4783        query_name: &str,
4784        input: T,
4785    ) -> Result<AvroValue> {
4786        self.client
4787            .query_workflow_avro_value(&self.workflow_id, query_name, input)
4788            .await
4789    }
4790
4791    pub async fn update<T: Serialize>(
4792        &self,
4793        update_name: &str,
4794        input: T,
4795        request_id: Option<&str>,
4796    ) -> Result<Value> {
4797        self.client
4798            .update_workflow(&self.workflow_id, update_name, input, request_id)
4799            .await
4800    }
4801
4802    pub async fn update_avro_value<T: Serialize>(
4803        &self,
4804        update_name: &str,
4805        input: T,
4806        request_id: Option<&str>,
4807    ) -> Result<AvroValue> {
4808        self.client
4809            .update_workflow_avro_value(&self.workflow_id, update_name, input, request_id)
4810            .await
4811    }
4812
4813    /// Query only if this handle's selected run is still current.
4814    pub async fn query_selected_run<T: Serialize>(
4815        &self,
4816        query_name: &str,
4817        input: T,
4818    ) -> Result<Value> {
4819        let run_id = self
4820            .run_id
4821            .as_deref()
4822            .ok_or_else(|| Error::Codec("run_id is required for selected-run query".to_string()))?;
4823        self.client
4824            .query_workflow_run(&self.workflow_id, run_id, query_name, input)
4825            .await
4826    }
4827
4828    /// Await the final terminal outcome of the current continue-as-new chain.
4829    pub async fn result(&self, options: WorkflowResultOptions) -> Result<Value> {
4830        self.result_target(options, None).await
4831    }
4832
4833    /// Await the final result without projecting Avro bytes through JSON.
4834    pub async fn result_avro_value(&self, options: WorkflowResultOptions) -> Result<AvroValue> {
4835        self.result_avro_value_target(options, None).await
4836    }
4837
4838    /// Await the final result and decode it into a Serde application type.
4839    pub async fn result_typed<T: DeserializeOwned>(
4840        &self,
4841        options: WorkflowResultOptions,
4842    ) -> Result<T> {
4843        let result = self.result_avro_value(options).await?;
4844        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4845    }
4846
4847    /// Await only the run identity originally selected by this handle.
4848    pub async fn result_selected_run(&self, options: WorkflowResultOptions) -> Result<Value> {
4849        let run_id = self.run_id.as_deref().ok_or_else(|| {
4850            Error::Codec("run_id is required for selected-run result".to_string())
4851        })?;
4852        self.result_target(options, Some(run_id)).await
4853    }
4854
4855    /// Await the selected run's result on the lossless Avro Value surface.
4856    pub async fn result_selected_run_avro_value(
4857        &self,
4858        options: WorkflowResultOptions,
4859    ) -> Result<AvroValue> {
4860        let run_id = self.run_id.as_deref().ok_or_else(|| {
4861            Error::Codec("run_id is required for selected-run result".to_string())
4862        })?;
4863        self.result_avro_value_target(options, Some(run_id)).await
4864    }
4865
4866    /// Await the selected run and decode its result into a Serde type.
4867    pub async fn result_selected_run_typed<T: DeserializeOwned>(
4868        &self,
4869        options: WorkflowResultOptions,
4870    ) -> Result<T> {
4871        let result = self.result_selected_run_avro_value(options).await?;
4872        decode_handler_result(result, HandlerKind::Workflow, &self.workflow_type)
4873    }
4874
4875    async fn result_avro_value_target(
4876        &self,
4877        options: WorkflowResultOptions,
4878        selected_run_id: Option<&str>,
4879    ) -> Result<AvroValue> {
4880        let started = Instant::now();
4881
4882        loop {
4883            let description = match selected_run_id {
4884                Some(run_id) => {
4885                    self.client
4886                        .describe_workflow_run(&self.workflow_id, run_id)
4887                        .await?
4888                }
4889                None => self.describe().await?,
4890            };
4891            if description.is_completed() {
4892                return description.output_avro_value.ok_or_else(|| {
4893                    Error::Codec(
4894                        "missing_payload_envelope: typed workflow result requires output_envelope"
4895                            .to_string(),
4896                    )
4897                });
4898            }
4899            if description.is_terminal() {
4900                let outcome =
4901                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4902                return Err(match outcome.kind {
4903                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4904                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4905                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4906                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4907                });
4908            }
4909            if started.elapsed() >= options.timeout {
4910                return Err(Error::Timeout);
4911            }
4912            tokio::time::sleep(options.poll_interval).await;
4913        }
4914    }
4915
4916    async fn result_target(
4917        &self,
4918        options: WorkflowResultOptions,
4919        selected_run_id: Option<&str>,
4920    ) -> Result<Value> {
4921        let started = Instant::now();
4922
4923        loop {
4924            let description = match selected_run_id {
4925                Some(run_id) => {
4926                    self.client
4927                        .describe_workflow_run(&self.workflow_id, run_id)
4928                        .await?
4929                }
4930                None => self.describe().await?,
4931            };
4932            if description.is_completed() {
4933                return Ok(description.output.unwrap_or(Value::Null));
4934            }
4935
4936            if description.is_terminal() {
4937                let outcome =
4938                    workflow_terminal_outcome(&description, &self.workflow_id, selected_run_id);
4939                return Err(match outcome.kind {
4940                    WorkflowTerminalKind::Failed => Error::WorkflowFailed(outcome),
4941                    WorkflowTerminalKind::Cancelled => Error::WorkflowCancelled(outcome),
4942                    WorkflowTerminalKind::Terminated => Error::WorkflowTerminated(outcome),
4943                    WorkflowTerminalKind::TimedOut => Error::WorkflowTimedOut(outcome),
4944                });
4945            }
4946
4947            if started.elapsed() >= options.timeout {
4948                return Err(Error::WorkflowTimedOut(WorkflowTerminalOutcome {
4949                    kind: WorkflowTerminalKind::TimedOut,
4950                    workflow_id: description
4951                        .workflow_id
4952                        .clone()
4953                        .unwrap_or_else(|| self.workflow_id.clone()),
4954                    run_id: description
4955                        .run_id
4956                        .clone()
4957                        .or_else(|| selected_run_id.map(str::to_string)),
4958                    reason: "result_wait_timeout".to_string(),
4959                    failure_category: Some("client_timeout".to_string()),
4960                    failure_id: None,
4961                    exception_type: None,
4962                    exception_class: None,
4963                    non_retryable: None,
4964                    message: Some(format!(
4965                        "workflow result was not terminal within {:?}",
4966                        options.timeout
4967                    )),
4968                    exception: None,
4969                    raw: description.raw_value(),
4970                }));
4971            }
4972
4973            tokio::time::sleep(options.poll_interval).await;
4974        }
4975    }
4976}
4977
4978#[derive(Clone, Copy, Debug)]
4979pub struct WorkflowResultOptions {
4980    pub poll_interval: Duration,
4981    pub timeout: Duration,
4982}
4983
4984impl Default for WorkflowResultOptions {
4985    fn default() -> Self {
4986        Self {
4987            poll_interval: Duration::from_millis(500),
4988            timeout: Duration::from_secs(30),
4989        }
4990    }
4991}
4992
4993#[derive(Clone, Debug, Deserialize)]
4994pub struct WorkflowDescription {
4995    pub workflow_id: Option<String>,
4996    pub run_id: Option<String>,
4997    pub workflow_type: Option<String>,
4998    pub status: Option<String>,
4999    #[serde(default)]
5000    pub closed_reason: Option<String>,
5001    #[serde(default)]
5002    pub error: Option<String>,
5003    #[serde(default)]
5004    pub failure: Option<Value>,
5005    #[serde(default)]
5006    pub exception: Option<Value>,
5007    #[serde(default)]
5008    pub failures: Vec<Value>,
5009    #[serde(default)]
5010    pub output: Option<Value>,
5011    #[serde(default)]
5012    pub output_envelope: Option<Value>,
5013    #[serde(skip)]
5014    pub output_avro_value: Option<AvroValue>,
5015    #[serde(flatten)]
5016    pub raw: HashMap<String, Value>,
5017}
5018
5019/// Lifecycle and backlog metadata for one run-scoped Workflow Stream.
5020#[derive(Clone, Debug, Deserialize)]
5021pub struct WorkflowStreamDescription {
5022    pub stream_name: String,
5023    pub status: String,
5024    pub last_offset: i64,
5025    pub total_items: u64,
5026    pub pending_items: u64,
5027    #[serde(default)]
5028    pub opened_at: Option<String>,
5029    #[serde(default)]
5030    pub last_appended_at: Option<String>,
5031    #[serde(default)]
5032    pub closed_at: Option<String>,
5033    #[serde(default)]
5034    pub error_reason: Option<String>,
5035    #[serde(default)]
5036    pub retention_seconds: Option<u64>,
5037    #[serde(flatten)]
5038    pub raw: HashMap<String, Value>,
5039}
5040
5041impl WorkflowStreamDescription {
5042    pub fn is_terminal(&self) -> bool {
5043        matches!(self.status.as_str(), "closed" | "errored")
5044    }
5045}
5046
5047/// One item for direct or replay-safe append.
5048#[derive(Clone, Debug, Default)]
5049pub struct WorkflowStreamAppendItem {
5050    pub payload_envelope: Option<Value>,
5051    pub payload_reference: Option<String>,
5052    pub item_type: Option<String>,
5053    pub content_type: Option<String>,
5054    pub idempotency_key: Option<String>,
5055}
5056
5057impl WorkflowStreamAppendItem {
5058    /// Encode an inline payload with the SDK's fixed Avro Value envelope.
5059    pub fn new<T: Serialize>(payload: T) -> Result<Self> {
5060        let value = AvroValue::from_serialize(&payload)?;
5061        Ok(Self {
5062            payload_envelope: Some(encode_typed_envelope(&value, DEFAULT_CODEC)?),
5063            ..Self::default()
5064        })
5065    }
5066
5067    /// Preserve an external payload URI as an opaque service-contract reference.
5068    pub fn from_reference(reference: impl Into<String>) -> Self {
5069        Self {
5070            payload_reference: Some(reference.into()),
5071            ..Self::default()
5072        }
5073    }
5074
5075    pub fn item_type(mut self, item_type: impl Into<String>) -> Self {
5076        self.item_type = Some(item_type.into());
5077        self
5078    }
5079
5080    pub fn content_type(mut self, content_type: impl Into<String>) -> Self {
5081        self.content_type = Some(content_type.into());
5082        self
5083    }
5084
5085    pub fn idempotency_key(mut self, idempotency_key: impl Into<String>) -> Self {
5086        self.idempotency_key = Some(idempotency_key.into());
5087        self
5088    }
5089
5090    fn wire_value(&self, derived_idempotency_key: Option<String>) -> Value {
5091        let mut item = serde_json::Map::new();
5092        if let Some(payload) = &self.payload_envelope {
5093            item.insert("payload".to_string(), payload.clone());
5094            item.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
5095        }
5096        if let Some(reference) = &self.payload_reference {
5097            item.insert("payload_reference".to_string(), json!(reference));
5098        }
5099        if let Some(item_type) = &self.item_type {
5100            item.insert("item_type".to_string(), json!(item_type));
5101        }
5102        if let Some(content_type) = &self.content_type {
5103            item.insert("content_type".to_string(), json!(content_type));
5104        }
5105        if let Some(key) = derived_idempotency_key
5106            .as_ref()
5107            .or(self.idempotency_key.as_ref())
5108        {
5109            item.insert("idempotency_key".to_string(), json!(key));
5110        }
5111        Value::Object(item)
5112    }
5113}
5114
5115/// One durable item at its stable zero-based offset.
5116#[derive(Clone, Debug)]
5117pub struct WorkflowStreamItem {
5118    pub offset: u64,
5119    pub payload: Option<Value>,
5120    pub payload_envelope: Option<Value>,
5121    pub payload_reference: Option<String>,
5122    pub payload_codec: Option<String>,
5123    pub idempotency_key: Option<String>,
5124    pub item_type: Option<String>,
5125    pub content_type: Option<String>,
5126    pub origin: Option<String>,
5127    pub origin_reference: Option<String>,
5128    pub emitted_at: Option<String>,
5129    pub raw: Value,
5130}
5131
5132/// One bounded at-least-once subscription page.
5133#[derive(Clone, Debug)]
5134pub struct WorkflowStreamPage {
5135    pub stream: WorkflowStreamDescription,
5136    pub items: Vec<WorkflowStreamItem>,
5137    pub next_offset: u64,
5138    pub terminal: bool,
5139}
5140
5141/// Durable acceptance and deduplication outcome for an append request.
5142#[derive(Clone, Debug)]
5143pub struct WorkflowStreamAppendResult {
5144    pub stream: WorkflowStreamDescription,
5145    pub accepted_offsets: Vec<u64>,
5146    pub accepted: u64,
5147    pub deduped: u64,
5148}
5149
5150#[derive(Deserialize)]
5151struct WorkflowStreamListResponse {
5152    #[serde(default)]
5153    streams: Vec<WorkflowStreamDescription>,
5154}
5155
5156#[derive(Deserialize)]
5157struct WorkflowStreamDescriptionResponse {
5158    stream: WorkflowStreamDescription,
5159}
5160
5161#[derive(Deserialize)]
5162struct WorkflowStreamPageResponse {
5163    stream: WorkflowStreamDescription,
5164    #[serde(default)]
5165    items: Vec<Value>,
5166    next_offset: u64,
5167    terminal: bool,
5168}
5169
5170#[derive(Deserialize)]
5171struct WorkflowStreamAppendResponse {
5172    stream: WorkflowStreamDescription,
5173    #[serde(default)]
5174    accepted_offsets: Vec<u64>,
5175    accepted: u64,
5176    deduped: u64,
5177}
5178
5179impl WorkflowDescription {
5180    pub fn is_completed(&self) -> bool {
5181        matches!(self.status.as_deref(), Some("completed" | "Completed"))
5182    }
5183
5184    pub fn is_terminal(&self) -> bool {
5185        matches!(
5186            self.status.as_deref(),
5187            Some(
5188                "completed"
5189                    | "Completed"
5190                    | "failed"
5191                    | "Failed"
5192                    | "cancelled"
5193                    | "Cancelled"
5194                    | "terminated"
5195                    | "Terminated"
5196                    | "timed_out"
5197                    | "TimedOut",
5198            )
5199        )
5200    }
5201
5202    fn decode_payloads(&mut self) -> Result<()> {
5203        if let Some(envelope) = &self.output_envelope {
5204            let value = decode_wire_avro_value(envelope, DEFAULT_CODEC)?;
5205            self.output = Some(value.clone().into_json()?);
5206            self.output_avro_value = Some(value);
5207        }
5208
5209        Ok(())
5210    }
5211
5212    fn raw_value(&self) -> Value {
5213        let mut data = self.raw.clone();
5214        data.insert(
5215            "workflow_id".to_string(),
5216            self.workflow_id
5217                .clone()
5218                .map(Value::String)
5219                .unwrap_or(Value::Null),
5220        );
5221        data.insert(
5222            "run_id".to_string(),
5223            self.run_id
5224                .clone()
5225                .map(Value::String)
5226                .unwrap_or(Value::Null),
5227        );
5228        data.insert(
5229            "workflow_type".to_string(),
5230            self.workflow_type
5231                .clone()
5232                .map(Value::String)
5233                .unwrap_or(Value::Null),
5234        );
5235        data.insert(
5236            "status".to_string(),
5237            self.status
5238                .clone()
5239                .map(Value::String)
5240                .unwrap_or(Value::Null),
5241        );
5242        data.insert(
5243            "closed_reason".to_string(),
5244            self.closed_reason
5245                .clone()
5246                .map(Value::String)
5247                .unwrap_or(Value::Null),
5248        );
5249        if let Some(failure) = &self.failure {
5250            data.insert("failure".to_string(), failure.clone());
5251        }
5252        if let Some(exception) = &self.exception {
5253            data.insert("exception".to_string(), exception.clone());
5254        }
5255        Value::Object(data.into_iter().collect())
5256    }
5257}
5258
5259fn workflow_terminal_outcome(
5260    description: &WorkflowDescription,
5261    workflow_id: &str,
5262    run_id: Option<&str>,
5263) -> WorkflowTerminalOutcome {
5264    let terminal_kind = description
5265        .closed_reason
5266        .as_deref()
5267        .or(description.status.as_deref())
5268        .unwrap_or("failed")
5269        .to_ascii_lowercase();
5270    let kind = match terminal_kind.as_str() {
5271        "cancelled" | "canceled" => WorkflowTerminalKind::Cancelled,
5272        "terminated" => WorkflowTerminalKind::Terminated,
5273        "timed_out" | "timedout" => WorkflowTerminalKind::TimedOut,
5274        _ => WorkflowTerminalKind::Failed,
5275    };
5276    let default_reason = match kind {
5277        WorkflowTerminalKind::Failed => "workflow_failed",
5278        WorkflowTerminalKind::Cancelled => "cancelled",
5279        WorkflowTerminalKind::Terminated => "terminated",
5280        WorkflowTerminalKind::TimedOut => "timed_out",
5281    };
5282    let failure = description
5283        .failure
5284        .as_ref()
5285        .filter(|value| value.is_object());
5286    let nested_failure = failure
5287        .and_then(|value| value.get("failures"))
5288        .and_then(Value::as_array)
5289        .and_then(|failures| failures.last())
5290        .or_else(|| description.failures.last());
5291    let exception = description
5292        .exception
5293        .clone()
5294        .or_else(|| failure.and_then(|value| value.get("exception")).cloned())
5295        .or_else(|| {
5296            nested_failure
5297                .and_then(|value| value.get("exception_payload"))
5298                .cloned()
5299        });
5300    let string_field = |name: &str| {
5301        failure
5302            .and_then(|value| value.get(name))
5303            .and_then(Value::as_str)
5304            .or_else(|| {
5305                nested_failure
5306                    .and_then(|value| value.get(name))
5307                    .and_then(Value::as_str)
5308            })
5309            .map(str::to_string)
5310    };
5311    let exception_field = |name: &str| {
5312        exception
5313            .as_ref()
5314            .and_then(|value| value.get(name))
5315            .and_then(Value::as_str)
5316            .map(str::to_string)
5317    };
5318    let message = description
5319        .error
5320        .clone()
5321        .or_else(|| string_field("message"))
5322        .or_else(|| exception_field("message"));
5323    let reason = description
5324        .raw
5325        .get("reason")
5326        .and_then(Value::as_str)
5327        .map(str::to_string)
5328        .or_else(|| {
5329            failure
5330                .and_then(|value| value.get("reason"))
5331                .and_then(Value::as_str)
5332                .map(str::to_string)
5333        })
5334        .or_else(|| description.closed_reason.clone())
5335        .unwrap_or_else(|| default_reason.to_string());
5336    let failure_id = string_field("failure_id").or_else(|| {
5337        nested_failure
5338            .and_then(|value| value.get("id"))
5339            .and_then(Value::as_str)
5340            .map(str::to_string)
5341    });
5342
5343    WorkflowTerminalOutcome {
5344        kind,
5345        workflow_id: description
5346            .workflow_id
5347            .clone()
5348            .unwrap_or_else(|| workflow_id.to_string()),
5349        run_id: description
5350            .run_id
5351            .clone()
5352            .or_else(|| run_id.map(str::to_string)),
5353        reason,
5354        failure_category: string_field("failure_category")
5355            .or_else(|| Some(default_reason.to_string())),
5356        failure_id,
5357        exception_type: string_field("exception_type").or_else(|| exception_field("type")),
5358        exception_class: string_field("exception_class").or_else(|| exception_field("class")),
5359        non_retryable: failure
5360            .and_then(|value| value.get("non_retryable"))
5361            .and_then(Value::as_bool)
5362            .or_else(|| {
5363                nested_failure
5364                    .and_then(|value| value.get("non_retryable"))
5365                    .and_then(Value::as_bool)
5366            }),
5367        message,
5368        exception,
5369        raw: description.raw_value(),
5370    }
5371}
5372
5373#[derive(Clone, Debug, Deserialize)]
5374pub struct RegisterWorkerResponse {
5375    pub worker_id: String,
5376    pub registered: bool,
5377    #[serde(default)]
5378    pub heartbeat_interval_seconds: Option<u64>,
5379    #[serde(default)]
5380    pub protocol_version: Option<String>,
5381    #[serde(default)]
5382    pub server_capabilities: Option<Value>,
5383}
5384
5385/// Result of gracefully removing a worker-plane registration.
5386#[derive(Clone, Debug, Deserialize, PartialEq, Eq)]
5387pub struct WorkerDeregistrationEnvelope {
5388    pub worker_id: String,
5389    pub outcome: String,
5390    pub recovered_workflow_task_count: u64,
5391}
5392
5393#[derive(Clone, Debug, Deserialize)]
5394pub struct PollWorkflowTaskResponse {
5395    #[serde(default)]
5396    pub task: Option<WorkflowTask>,
5397    #[serde(default)]
5398    pub poll_status: Option<String>,
5399    #[serde(default)]
5400    pub reason: Option<String>,
5401    #[serde(default)]
5402    pub protocol_version: Option<String>,
5403    #[serde(default)]
5404    pub server_capabilities: Option<Value>,
5405}
5406
5407impl PollWorkflowTaskResponse {
5408    /// Classify this response without parsing server display text.
5409    pub fn outcome(&self) -> WorkerPollOutcome {
5410        worker_poll_outcome(
5411            self.task.is_some(),
5412            self.poll_status.as_deref(),
5413            self.reason.as_deref(),
5414        )
5415    }
5416}
5417
5418fn runtime_supports_workflow_memo_updates(capabilities: Option<&Value>) -> bool {
5419    let Some(capabilities) = capabilities.and_then(Value::as_object) else {
5420        return false;
5421    };
5422    let supported = capabilities
5423        .get("workflow_memo_updates")
5424        .and_then(Value::as_object)
5425        .and_then(|memo| memo.get("supported"))
5426        .and_then(Value::as_bool)
5427        == Some(true);
5428    let command_advertised = capabilities
5429        .get("supported_workflow_task_commands")
5430        .and_then(Value::as_array)
5431        .is_some_and(|commands| {
5432            commands
5433                .iter()
5434                .any(|command| command.as_str() == Some("upsert_memo"))
5435        });
5436    supported && command_advertised
5437}
5438
5439fn commands_use_workflow_memo_updates(commands: &[Value]) -> bool {
5440    commands
5441        .iter()
5442        .any(|command| command.get("type").and_then(Value::as_str) == Some("upsert_memo"))
5443}
5444
5445#[derive(Clone, Debug, Deserialize)]
5446pub struct PollActivityTaskResponse {
5447    #[serde(default)]
5448    pub task: Option<ActivityTask>,
5449    #[serde(default)]
5450    pub poll_status: Option<String>,
5451    #[serde(default)]
5452    pub reason: Option<String>,
5453}
5454
5455impl PollActivityTaskResponse {
5456    /// Classify this response without parsing server display text.
5457    pub fn outcome(&self) -> WorkerPollOutcome {
5458        worker_poll_outcome(
5459            self.task.is_some(),
5460            self.poll_status.as_deref(),
5461            self.reason.as_deref(),
5462        )
5463    }
5464}
5465
5466#[derive(Clone, Debug, Deserialize)]
5467pub struct PollQueryTaskResponse {
5468    #[serde(default)]
5469    pub task: Option<QueryTask>,
5470    #[serde(default)]
5471    pub poll_status: Option<String>,
5472    #[serde(default)]
5473    pub reason: Option<String>,
5474}
5475
5476impl PollQueryTaskResponse {
5477    /// Classify this response without parsing server display text.
5478    pub fn outcome(&self) -> WorkerPollOutcome {
5479        worker_poll_outcome(
5480            self.task.is_some(),
5481            self.poll_status.as_deref(),
5482            self.reason.as_deref(),
5483        )
5484    }
5485}
5486
5487/// Stable classification for worker poll responses.
5488#[derive(Clone, Debug, PartialEq, Eq)]
5489pub enum WorkerPollOutcome {
5490    /// A task was leased and is available on the response.
5491    Task,
5492    /// No task was leased, but the worker should continue polling.
5493    Idle {
5494        poll_status: Option<String>,
5495        reason: Option<String>,
5496    },
5497    /// The server asked this worker to stop claiming new work.
5498    Stop {
5499        poll_status: Option<String>,
5500        reason: Option<String>,
5501    },
5502}
5503
5504impl WorkerPollOutcome {
5505    pub fn should_stop(&self) -> bool {
5506        matches!(self, Self::Stop { .. })
5507    }
5508}
5509
5510fn worker_poll_outcome(
5511    has_task: bool,
5512    poll_status: Option<&str>,
5513    reason: Option<&str>,
5514) -> WorkerPollOutcome {
5515    if worker_poll_is_stop(poll_status, reason) {
5516        return WorkerPollOutcome::Stop {
5517            poll_status: poll_status.map(str::to_string),
5518            reason: reason.map(str::to_string),
5519        };
5520    }
5521
5522    if has_task {
5523        WorkerPollOutcome::Task
5524    } else {
5525        WorkerPollOutcome::Idle {
5526            poll_status: poll_status.map(str::to_string),
5527            reason: reason.map(str::to_string),
5528        }
5529    }
5530}
5531
5532/// An ephemeral server-routed query task.
5533#[derive(Clone, Debug, Deserialize)]
5534pub struct QueryTask {
5535    pub query_task_id: String,
5536    #[serde(default = "default_workflow_task_attempt")]
5537    pub query_task_attempt: u64,
5538    #[serde(default)]
5539    pub lease_owner: Option<String>,
5540    #[serde(default)]
5541    pub workflow_id: Option<String>,
5542    #[serde(default)]
5543    pub run_id: Option<String>,
5544    pub workflow_type: String,
5545    pub query_name: String,
5546    #[serde(
5547        default = "missing_task_payload_codec",
5548        deserialize_with = "deserialize_task_payload_codec"
5549    )]
5550    pub payload_codec: String,
5551    #[serde(default)]
5552    pub workflow_arguments: Option<Value>,
5553    #[serde(default)]
5554    pub query_arguments: Option<Value>,
5555    #[serde(default)]
5556    pub history_events: Vec<HistoryEvent>,
5557    #[serde(default)]
5558    pub history_export: Option<Value>,
5559    #[serde(default)]
5560    pub run_status: Option<String>,
5561}
5562
5563#[derive(Clone, Debug, Deserialize)]
5564pub struct WorkflowTask {
5565    pub task_id: String,
5566    #[serde(default)]
5567    pub workflow_command_id: Option<String>,
5568    #[serde(default)]
5569    pub workflow_id: Option<String>,
5570    #[serde(default)]
5571    pub run_id: Option<String>,
5572    pub workflow_type: String,
5573    #[serde(default)]
5574    pub cancel_requested: bool,
5575    #[serde(
5576        default = "missing_task_payload_codec",
5577        deserialize_with = "deserialize_task_payload_codec"
5578    )]
5579    pub payload_codec: String,
5580    #[serde(default)]
5581    pub arguments: Option<Value>,
5582    #[serde(default)]
5583    pub history_events: Vec<HistoryEvent>,
5584    #[serde(default)]
5585    pub total_history_events: Option<u64>,
5586    #[serde(default)]
5587    pub history_size_bytes: Option<u64>,
5588    #[serde(default)]
5589    pub continue_as_new_recommended: Option<bool>,
5590    #[serde(default)]
5591    pub history_budget_pressure: Option<String>,
5592    #[serde(default)]
5593    pub next_history_page_token: Option<String>,
5594    #[serde(default = "default_workflow_task_attempt")]
5595    pub workflow_task_attempt: u64,
5596    #[serde(default)]
5597    pub workflow_signal_id: Option<String>,
5598    #[serde(default)]
5599    pub signal_name: Option<String>,
5600    #[serde(default)]
5601    pub signal_arguments: Option<Value>,
5602    #[serde(default)]
5603    pub workflow_update_id: Option<String>,
5604    #[serde(default)]
5605    pub update_name: Option<String>,
5606    #[serde(default)]
5607    pub lease_owner: Option<String>,
5608}
5609
5610impl WorkflowTask {
5611    fn append_history_page(&mut self, page: WorkflowTaskHistoryPage) {
5612        self.history_events.extend(page.history_events);
5613
5614        if page.total_history_events.is_some() {
5615            self.total_history_events = page.total_history_events;
5616        }
5617
5618        self.next_history_page_token = page
5619            .next_history_page_token
5620            .filter(|token| !token.is_empty());
5621    }
5622}
5623
5624#[derive(Clone, Debug, Deserialize)]
5625struct WorkflowTaskHistoryPage {
5626    #[serde(default)]
5627    history_events: Vec<HistoryEvent>,
5628    #[serde(default)]
5629    total_history_events: Option<u64>,
5630    #[serde(default)]
5631    next_history_page_token: Option<String>,
5632}
5633
5634#[derive(Clone, Debug, Deserialize)]
5635pub struct ActivityTask {
5636    pub task_id: String,
5637    #[serde(default)]
5638    pub activity_attempt_id: Option<String>,
5639    #[serde(default)]
5640    pub attempt_id: Option<String>,
5641    pub activity_type: String,
5642    #[serde(
5643        default = "missing_task_payload_codec",
5644        deserialize_with = "deserialize_task_payload_codec"
5645    )]
5646    pub payload_codec: String,
5647    #[serde(default)]
5648    pub arguments: Option<Value>,
5649    #[serde(default = "default_attempt_number")]
5650    pub attempt_number: u64,
5651    #[serde(default)]
5652    pub lease_owner: Option<String>,
5653}
5654
5655#[derive(Clone, Debug, Deserialize)]
5656pub struct HistoryEvent {
5657    #[serde(alias = "type")]
5658    pub event_type: String,
5659    #[serde(default)]
5660    pub payload: Value,
5661    #[serde(flatten)]
5662    pub raw: HashMap<String, Value>,
5663}
5664
5665/// One decoded signal in the committed workflow-history snapshot.
5666#[derive(Clone, Debug, PartialEq)]
5667pub struct QuerySignal {
5668    pub id: Option<String>,
5669    pub name: String,
5670    pub arguments: Vec<Value>,
5671    avro_arguments: Vec<AvroValue>,
5672    pub workflow_sequence: Option<u64>,
5673}
5674
5675impl QuerySignal {
5676    /// Lossless fixed Avro Value arguments for this committed signal.
5677    pub fn arguments_avro_value(&self) -> &[AvroValue] {
5678        &self.avro_arguments
5679    }
5680}
5681
5682/// Immutable state supplied to a registered query handler.
5683///
5684/// This context intentionally exposes no activity, signal-wait, or command
5685/// APIs. Query handlers inspect committed history and return a value; query
5686/// completion does not append an event or advance deterministic execution.
5687#[derive(Clone, Debug)]
5688pub struct QueryContext {
5689    pub workflow_id: Option<String>,
5690    pub run_id: Option<String>,
5691    pub workflow_type: String,
5692    pub run_status: Option<String>,
5693    workflow_input: Value,
5694    workflow_input_avro_value: AvroValue,
5695    history_events: Arc<Vec<HistoryEvent>>,
5696    signal_events: Arc<Vec<QuerySignal>>,
5697}
5698
5699impl QueryContext {
5700    /// The normalized argument list used to start the workflow.
5701    pub fn workflow_input(&self) -> &Value {
5702        &self.workflow_input
5703    }
5704
5705    /// The lossless fixed Avro Value argument list used to start the workflow.
5706    pub fn workflow_input_avro_value(&self) -> &AvroValue {
5707        &self.workflow_input_avro_value
5708    }
5709
5710    /// The immutable committed history used for this query snapshot.
5711    pub fn history_events(&self) -> &[HistoryEvent] {
5712        self.history_events.as_slice()
5713    }
5714
5715    /// All decoded signals in committed workflow order.
5716    pub fn signal_events(&self) -> &[QuerySignal] {
5717        self.signal_events.as_slice()
5718    }
5719
5720    /// Decoded argument lists for each committed signal with `signal_name`.
5721    pub fn signals(&self, signal_name: &str) -> Vec<Vec<Value>> {
5722        self.signal_events
5723            .iter()
5724            .filter(|signal| signal.name == signal_name)
5725            .map(|signal| signal.arguments.clone())
5726            .collect()
5727    }
5728
5729    /// Lossless fixed Avro Value arguments for committed signals with `signal_name`.
5730    pub fn signals_avro_value(&self, signal_name: &str) -> Vec<Vec<AvroValue>> {
5731        self.signal_events
5732            .iter()
5733            .filter(|signal| signal.name == signal_name)
5734            .map(|signal| signal.avro_arguments.clone())
5735            .collect()
5736    }
5737}
5738
5739#[derive(Clone, Debug, Deserialize)]
5740pub struct ActivityHeartbeatResponse {
5741    #[serde(default)]
5742    pub cancel_requested: bool,
5743    #[serde(default)]
5744    pub heartbeat_recorded: bool,
5745    #[serde(default)]
5746    pub can_continue: Option<bool>,
5747    #[serde(default)]
5748    pub reason: Option<String>,
5749    #[serde(default)]
5750    pub run_closed_reason: Option<String>,
5751    #[serde(default)]
5752    pub run_closed_at: Option<String>,
5753    #[serde(default)]
5754    pub lease_expires_at: Option<String>,
5755    #[serde(default)]
5756    pub last_heartbeat_at: Option<String>,
5757}
5758
5759impl ActivityHeartbeatResponse {
5760    /// Whether the activity should stop instead of attempting completion.
5761    pub fn should_stop(&self) -> bool {
5762        self.cancel_requested || self.can_continue == Some(false)
5763    }
5764}
5765
5766fn missing_task_payload_codec() -> String {
5767    MISSING_TASK_PAYLOAD_CODEC.to_string()
5768}
5769
5770fn deserialize_task_payload_codec<'de, D>(deserializer: D) -> std::result::Result<String, D::Error>
5771where
5772    D: Deserializer<'de>,
5773{
5774    Ok(match Value::deserialize(deserializer)? {
5775        Value::String(codec) => codec,
5776        Value::Null => NULL_TASK_PAYLOAD_CODEC.to_string(),
5777        _ => NON_STRING_TASK_PAYLOAD_CODEC.to_string(),
5778    })
5779}
5780
5781fn default_workflow_task_attempt() -> u64 {
5782    1
5783}
5784
5785fn default_attempt_number() -> u64 {
5786    1
5787}
5788
5789type WorkflowFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5790type WorkflowHandler = Arc<dyn Fn(WorkflowContext, AvroValue) -> WorkflowFuture + Send + Sync>;
5791type ErasedWorkflowState = Arc<dyn Any + Send + Sync>;
5792type WorkflowStateSnapshot = Arc<dyn Fn() -> Result<ErasedWorkflowState> + Send + Sync>;
5793type ReplayedWorkflowHandler =
5794    Arc<dyn Fn(WorkflowContext, AvroValue) -> ReplayedWorkflowInvocation + Send + Sync>;
5795type ActivityFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5796type ActivityHandler = Arc<dyn Fn(ActivityContext, AvroValue) -> ActivityFuture + Send + Sync>;
5797type QueryFuture = Pin<Box<dyn Future<Output = Result<AvroValue>> + Send + 'static>>;
5798type QueryHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5799type UpdateHandler = Arc<dyn Fn(QueryContext, AvroValue) -> QueryFuture + Send + Sync>;
5800type ReplayedQueryHandler = Arc<
5801    dyn Fn(QueryContext, ErasedWorkflowState, AvroValue) -> std::result::Result<QueryFuture, String>
5802        + Send
5803        + Sync,
5804>;
5805type WorkerHeartbeatObserver = Arc<dyn Fn(&WorkerHeartbeatObservation) + Send + Sync>;
5806
5807struct ReplayedWorkflowInvocation {
5808    future: WorkflowFuture,
5809    snapshot: WorkflowStateSnapshot,
5810}
5811
5812#[derive(Clone)]
5813struct RegisteredWorkflow {
5814    execute: WorkflowHandler,
5815    replay: Option<ReplayedWorkflowHandler>,
5816    state_type: Option<TypeId>,
5817}
5818
5819#[derive(Debug)]
5820struct WorkflowTaskDecision {
5821    commands: Vec<Value>,
5822    message_stream_cursors: Vec<Value>,
5823    message_stream_waits: Vec<Value>,
5824}
5825
5826impl WorkflowTaskDecision {
5827    fn without_message_streams(commands: Vec<Value>) -> Self {
5828        Self {
5829            commands,
5830            message_stream_cursors: Vec::new(),
5831            message_stream_waits: Vec::new(),
5832        }
5833    }
5834}
5835
5836#[derive(Clone)]
5837enum RegisteredQuery {
5838    Snapshot(QueryHandler),
5839    Replayed {
5840        state_type: TypeId,
5841        handler: ReplayedQueryHandler,
5842    },
5843}
5844
5845#[derive(Clone, Debug)]
5846pub struct WorkerHeartbeatObservation {
5847    pub worker_id: String,
5848    pub task_queue: String,
5849    pub acknowledged_at_unix_millis: u64,
5850    pub acknowledgement: Value,
5851}
5852
5853/// Bounded retry policy for worker poll acquisition and worker heartbeats.
5854///
5855/// Expected empty long polls and explicit Server capacity backpressure are
5856/// normal worker states. Capacity backpressure honors the advertised retry
5857/// delay without consuming this retry budget. Other transport failures,
5858/// retryable HTTP 408/429 responses, and server errors are retried with capped
5859/// exponential backoff. Authentication, protocol, codec, and handler failures
5860/// are never retried by the worker.
5861/// Explicit storage admission pauses also preserve the already-serialized worker
5862/// request without consuming this budget or rerunning its handler. The delay is
5863/// capped by `max_backoff`, and shutdown interrupts storage waits.
5864#[derive(Clone, Copy, Debug)]
5865pub struct WorkerRetryPolicy {
5866    /// Number of retries after the initial request fails.
5867    pub max_retries: usize,
5868    /// Delay before the first retry.
5869    pub initial_backoff: Duration,
5870    /// Maximum delay between retries.
5871    pub max_backoff: Duration,
5872}
5873
5874impl Default for WorkerRetryPolicy {
5875    fn default() -> Self {
5876        Self {
5877            max_retries: 5,
5878            initial_backoff: Duration::from_millis(100),
5879            max_backoff: Duration::from_secs(5),
5880        }
5881    }
5882}
5883
5884#[derive(Clone, Debug)]
5885struct WorkerStorageAdmission {
5886    policy: WorkerRetryPolicy,
5887    stop: Arc<AtomicBool>,
5888}
5889
5890struct StopWorkerOnDrop(Arc<AtomicBool>);
5891
5892impl Drop for StopWorkerOnDrop {
5893    fn drop(&mut self) {
5894        self.0.store(true, Ordering::SeqCst);
5895    }
5896}
5897
5898async fn wait_for_worker_stop(stop: &AtomicBool) {
5899    while !stop.load(Ordering::SeqCst) {
5900        tokio::time::sleep(Duration::from_millis(100)).await;
5901    }
5902}
5903
5904#[derive(Clone, Copy, Debug, PartialEq, Eq)]
5905enum ManagedPollOutcome {
5906    Idle,
5907    Handled,
5908    Stop,
5909}
5910
5911#[derive(Clone)]
5912pub struct Worker {
5913    client: Client,
5914    worker_id: String,
5915    task_queue: String,
5916    workflows: HashMap<String, RegisteredWorkflow>,
5917    activities: HashMap<String, ActivityHandler>,
5918    queries: HashMap<String, HashMap<String, RegisteredQuery>>,
5919    updates: HashMap<String, HashMap<String, UpdateHandler>>,
5920    max_concurrent_workflow_tasks: usize,
5921    max_concurrent_activity_tasks: usize,
5922    poll_timeout: Duration,
5923    heartbeat_interval: Duration,
5924    retry_policy: WorkerRetryPolicy,
5925    heartbeat_observer: Option<WorkerHeartbeatObserver>,
5926}
5927
5928impl Worker {
5929    pub fn new(client: Client, task_queue: impl Into<String>) -> Self {
5930        Self {
5931            client,
5932            worker_id: default_worker_id(),
5933            task_queue: task_queue.into(),
5934            workflows: HashMap::new(),
5935            activities: HashMap::new(),
5936            queries: HashMap::new(),
5937            updates: HashMap::new(),
5938            max_concurrent_workflow_tasks: 10,
5939            max_concurrent_activity_tasks: 10,
5940            poll_timeout: Duration::from_secs(30),
5941            heartbeat_interval: Duration::from_secs(60),
5942            retry_policy: WorkerRetryPolicy::default(),
5943            heartbeat_observer: None,
5944        }
5945    }
5946
5947    pub fn worker_id(mut self, worker_id: impl Into<String>) -> Self {
5948        self.worker_id = worker_id.into();
5949        self
5950    }
5951
5952    pub fn poll_timeout(mut self, timeout: Duration) -> Self {
5953        self.poll_timeout = timeout;
5954        self
5955    }
5956
5957    pub fn heartbeat_interval(mut self, interval: Duration) -> Self {
5958        self.heartbeat_interval = interval;
5959        self
5960    }
5961
5962    /// Configure bounded retries for task-poll acquisition and worker heartbeats.
5963    pub fn retry_policy(mut self, policy: WorkerRetryPolicy) -> Self {
5964        self.retry_policy = policy;
5965        self
5966    }
5967
5968    pub fn on_worker_heartbeat<F>(mut self, observer: F) -> Self
5969    where
5970        F: Fn(&WorkerHeartbeatObservation) + Send + Sync + 'static,
5971    {
5972        self.heartbeat_observer = Some(Arc::new(observer));
5973        self
5974    }
5975
5976    pub fn max_concurrent_workflow_tasks(mut self, count: usize) -> Self {
5977        self.max_concurrent_workflow_tasks = count.max(1);
5978        self
5979    }
5980
5981    pub fn max_concurrent_activity_tasks(mut self, count: usize) -> Self {
5982        self.max_concurrent_activity_tasks = count.max(1);
5983        self
5984    }
5985
5986    /// Register a workflow handler.
5987    ///
5988    /// An uncaught [`enum@Error`] returned by the handler fails the workflow run and
5989    /// is reported to clients as [`Error::WorkflowFailed`]. Errors that occur
5990    /// while acquiring or decoding a worker task remain worker-operation
5991    /// failures and do not get converted into workflow outcomes.
5992    pub fn register_workflow<F, Fut>(&mut self, workflow_type: impl Into<String>, handler: F)
5993    where
5994        F: Fn(WorkflowContext, Value) -> Fut + Send + Sync + 'static,
5995        Fut: Future<Output = Result<Value>> + Send + 'static,
5996    {
5997        let handler = Arc::new(handler);
5998        self.workflows.insert(
5999            workflow_type.into(),
6000            RegisteredWorkflow {
6001                execute: Arc::new(move |ctx, input| {
6002                    let handler = Arc::clone(&handler);
6003                    Box::pin(async move {
6004                        let result = handler(ctx, input.into_json()?).await?;
6005                        AvroValue::from_serialize(&result)
6006                    })
6007                }),
6008                replay: None,
6009                state_type: None,
6010            },
6011        );
6012    }
6013
6014    /// Register a workflow with one Serde request value and a Serde result.
6015    ///
6016    /// This is an ergonomic adapter over the same fixed Avro Value protocol as
6017    /// [`Worker::register_workflow_avro_value`]. It does not create or publish a
6018    /// workflow-specific schema. A task must contain zero arguments for a unit
6019    /// request or exactly one argument for every other request type.
6020    ///
6021    /// See the runnable
6022    /// [`hello_world` example](https://github.com/durable-workflow/sdk-rust/blob/main/examples/hello_world.rs)
6023    /// for typed workflow and activity contracts with retry and timeout policy.
6024    pub fn register_typed_workflow<I, O, F, Fut>(
6025        &mut self,
6026        workflow_type: impl Into<String>,
6027        handler: F,
6028    ) where
6029        I: DeserializeOwned + Send + 'static,
6030        O: Serialize + Send + 'static,
6031        F: Fn(WorkflowContext, I) -> Fut + Send + Sync + 'static,
6032        Fut: Future<Output = Result<O>> + Send + 'static,
6033    {
6034        let workflow_type = workflow_type.into();
6035        let handler_name = workflow_type.clone();
6036        let handler = Arc::new(handler);
6037        self.workflows.insert(
6038            workflow_type,
6039            RegisteredWorkflow {
6040                execute: Arc::new(move |ctx, input| {
6041                    let handler = Arc::clone(&handler);
6042                    let handler_name = handler_name.clone();
6043                    Box::pin(async move {
6044                        let input =
6045                            decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6046                        let result = handler(ctx, input).await?;
6047                        encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6048                    })
6049                }),
6050                replay: None,
6051                state_type: None,
6052            },
6053        );
6054    }
6055
6056    /// Register a workflow on the lossless fixed Avro Value surface.
6057    pub fn register_workflow_avro_value<F, Fut>(
6058        &mut self,
6059        workflow_type: impl Into<String>,
6060        handler: F,
6061    ) where
6062        F: Fn(WorkflowContext, AvroValue) -> Fut + Send + Sync + 'static,
6063        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6064    {
6065        self.workflows.insert(
6066            workflow_type.into(),
6067            RegisteredWorkflow {
6068                execute: Arc::new(move |ctx, input| Box::pin(handler(ctx, input))),
6069                replay: None,
6070                state_type: None,
6071            },
6072        );
6073    }
6074
6075    /// Register a workflow whose typed instance state can be reconstructed for queries.
6076    ///
6077    /// `state_factory` creates a fresh instance for every normal workflow task and
6078    /// query replay. The workflow handler is the single source of truth for state
6079    /// transitions: it updates [`WorkflowInstance`] after activities and signals
6080    /// resolve. Query replay runs this same handler over committed history and
6081    /// discards any commands it would emit.
6082    pub fn register_replayed_workflow<S, Factory, F, Fut>(
6083        &mut self,
6084        workflow_type: impl Into<String>,
6085        state_factory: Factory,
6086        handler: F,
6087    ) where
6088        S: Clone + Send + Sync + 'static,
6089        Factory: Fn() -> S + Send + Sync + 'static,
6090        F: Fn(WorkflowContext, Value, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6091        Fut: Future<Output = Result<Value>> + Send + 'static,
6092    {
6093        let state_factory = Arc::new(state_factory);
6094        let handler = Arc::new(handler);
6095
6096        let execute_factory = Arc::clone(&state_factory);
6097        let execute_handler = Arc::clone(&handler);
6098        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6099            let state = WorkflowInstance::new(execute_factory());
6100            let handler = Arc::clone(&execute_handler);
6101            Box::pin(async move {
6102                let result = handler(ctx, input.into_json()?, state).await?;
6103                AvroValue::from_serialize(&result)
6104            }) as WorkflowFuture
6105        });
6106
6107        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6108            let state = WorkflowInstance::new(state_factory());
6109            let snapshot_state = state.clone();
6110            let snapshot: WorkflowStateSnapshot =
6111                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6112            let replay_handler = Arc::clone(&handler);
6113            let future = async move {
6114                let result = replay_handler(ctx, input.into_json()?, state).await?;
6115                AvroValue::from_serialize(&result)
6116            };
6117            ReplayedWorkflowInvocation {
6118                future: Box::pin(future),
6119                snapshot,
6120            }
6121        });
6122
6123        self.workflows.insert(
6124            workflow_type.into(),
6125            RegisteredWorkflow {
6126                execute,
6127                replay: Some(replay),
6128                state_type: Some(TypeId::of::<S>()),
6129            },
6130        );
6131    }
6132
6133    /// Register a replayable workflow with one Serde request value and result.
6134    ///
6135    /// Normal task execution and instance-state query replay both decode and
6136    /// encode through the fixed Avro Value codec. The state factory and handler
6137    /// otherwise follow [`Worker::register_replayed_workflow`].
6138    pub fn register_typed_replayed_workflow<I, O, S, Factory, F, Fut>(
6139        &mut self,
6140        workflow_type: impl Into<String>,
6141        state_factory: Factory,
6142        handler: F,
6143    ) where
6144        I: DeserializeOwned + Send + 'static,
6145        O: Serialize + Send + 'static,
6146        S: Clone + Send + Sync + 'static,
6147        Factory: Fn() -> S + Send + Sync + 'static,
6148        F: Fn(WorkflowContext, I, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6149        Fut: Future<Output = Result<O>> + Send + 'static,
6150    {
6151        let workflow_type = workflow_type.into();
6152        let state_factory = Arc::new(state_factory);
6153        let handler = Arc::new(handler);
6154
6155        let execute_name = workflow_type.clone();
6156        let execute_factory = Arc::clone(&state_factory);
6157        let execute_handler = Arc::clone(&handler);
6158        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6159            let state = WorkflowInstance::new(execute_factory());
6160            let handler = Arc::clone(&execute_handler);
6161            let handler_name = execute_name.clone();
6162            Box::pin(async move {
6163                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6164                let result = handler(ctx, input, state).await?;
6165                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6166            }) as WorkflowFuture
6167        });
6168
6169        let replay_name = workflow_type.clone();
6170        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6171            let state = WorkflowInstance::new(state_factory());
6172            let snapshot_state = state.clone();
6173            let snapshot: WorkflowStateSnapshot =
6174                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6175            let handler = Arc::clone(&handler);
6176            let handler_name = replay_name.clone();
6177            let future = async move {
6178                let input = decode_handler_input::<I>(input, HandlerKind::Workflow, &handler_name)?;
6179                let result = handler(ctx, input, state).await?;
6180                encode_handler_result(&result, HandlerKind::Workflow, &handler_name)
6181            };
6182            ReplayedWorkflowInvocation {
6183                future: Box::pin(future),
6184                snapshot,
6185            }
6186        });
6187
6188        self.workflows.insert(
6189            workflow_type,
6190            RegisteredWorkflow {
6191                execute,
6192                replay: Some(replay),
6193                state_type: Some(TypeId::of::<S>()),
6194            },
6195        );
6196    }
6197
6198    /// Register a replayable workflow on the lossless fixed Avro Value surface.
6199    pub fn register_replayed_workflow_avro_value<S, Factory, F, Fut>(
6200        &mut self,
6201        workflow_type: impl Into<String>,
6202        state_factory: Factory,
6203        handler: F,
6204    ) where
6205        S: Clone + Send + Sync + 'static,
6206        Factory: Fn() -> S + Send + Sync + 'static,
6207        F: Fn(WorkflowContext, AvroValue, WorkflowInstance<S>) -> Fut + Send + Sync + 'static,
6208        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6209    {
6210        let state_factory = Arc::new(state_factory);
6211        let handler = Arc::new(handler);
6212
6213        let execute_factory = Arc::clone(&state_factory);
6214        let execute_handler = Arc::clone(&handler);
6215        let execute = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6216            let state = WorkflowInstance::new(execute_factory());
6217            Box::pin(execute_handler(ctx, input, state)) as WorkflowFuture
6218        });
6219
6220        let replay = Arc::new(move |ctx: WorkflowContext, input: AvroValue| {
6221            let state = WorkflowInstance::new(state_factory());
6222            let snapshot_state = state.clone();
6223            let snapshot: WorkflowStateSnapshot =
6224                Arc::new(move || Ok(Arc::new(snapshot_state.snapshot()?) as ErasedWorkflowState));
6225            ReplayedWorkflowInvocation {
6226                future: Box::pin(handler(ctx, input, state)),
6227                snapshot,
6228            }
6229        });
6230
6231        self.workflows.insert(
6232            workflow_type.into(),
6233            RegisteredWorkflow {
6234                execute,
6235                replay: Some(replay),
6236                state_type: Some(TypeId::of::<S>()),
6237            },
6238        );
6239    }
6240
6241    pub fn register_activity<F, Fut>(&mut self, activity_type: impl Into<String>, handler: F)
6242    where
6243        F: Fn(ActivityContext, Value) -> Fut + Send + Sync + 'static,
6244        Fut: Future<Output = Result<Value>> + Send + 'static,
6245    {
6246        let handler = Arc::new(handler);
6247        self.activities.insert(
6248            activity_type.into(),
6249            Arc::new(move |ctx, args| {
6250                let handler = Arc::clone(&handler);
6251                Box::pin(async move {
6252                    let result = handler(ctx, args.into_json()?).await?;
6253                    AvroValue::from_serialize(&result)
6254                })
6255            }),
6256        );
6257    }
6258
6259    /// Register an activity with one Serde request value and a Serde result.
6260    ///
6261    /// Inputs and results use the platform's fixed Avro Value schema. Shape
6262    /// mismatches and unsupported Serde values return [`Error::HandlerType`]
6263    /// with the activity name and Rust type.
6264    pub fn register_typed_activity<I, O, F, Fut>(
6265        &mut self,
6266        activity_type: impl Into<String>,
6267        handler: F,
6268    ) where
6269        I: DeserializeOwned + Send + 'static,
6270        O: Serialize + Send + 'static,
6271        F: Fn(ActivityContext, I) -> Fut + Send + Sync + 'static,
6272        Fut: Future<Output = Result<O>> + Send + 'static,
6273    {
6274        let activity_type = activity_type.into();
6275        let handler_name = activity_type.clone();
6276        let handler = Arc::new(handler);
6277        self.activities.insert(
6278            activity_type,
6279            Arc::new(move |ctx, input| {
6280                let handler = Arc::clone(&handler);
6281                let handler_name = handler_name.clone();
6282                Box::pin(async move {
6283                    let input =
6284                        decode_handler_input::<I>(input, HandlerKind::Activity, &handler_name)?;
6285                    let result = handler(ctx, input).await?;
6286                    encode_handler_result(&result, HandlerKind::Activity, &handler_name)
6287                })
6288            }),
6289        );
6290    }
6291
6292    /// Register an activity on the lossless fixed Avro Value surface.
6293    pub fn register_activity_avro_value<F, Fut>(
6294        &mut self,
6295        activity_type: impl Into<String>,
6296        handler: F,
6297    ) where
6298        F: Fn(ActivityContext, AvroValue) -> Fut + Send + Sync + 'static,
6299        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6300    {
6301        self.activities.insert(
6302            activity_type.into(),
6303            Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6304        );
6305    }
6306
6307    /// Register a named, read-only query handler for a workflow type.
6308    ///
6309    /// The workflow type must also be registered with [`Worker::register_workflow`]
6310    /// before the worker runs. The handler receives only an immutable committed
6311    /// state snapshot and normalized query arguments.
6312    pub fn register_query<F, Fut>(
6313        &mut self,
6314        workflow_type: impl Into<String>,
6315        query_name: impl Into<String>,
6316        handler: F,
6317    ) where
6318        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6319        Fut: Future<Output = Result<Value>> + Send + 'static,
6320    {
6321        let handler = Arc::new(handler);
6322        self.queries
6323            .entry(workflow_type.into())
6324            .or_default()
6325            .insert(
6326                query_name.into(),
6327                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| {
6328                    let handler = Arc::clone(&handler);
6329                    Box::pin(async move {
6330                        let result = handler(ctx, args.into_json()?).await?;
6331                        AvroValue::from_serialize(&result)
6332                    })
6333                })),
6334            );
6335    }
6336
6337    /// Register a query handler on the lossless fixed Avro Value surface.
6338    pub fn register_query_avro_value<F, Fut>(
6339        &mut self,
6340        workflow_type: impl Into<String>,
6341        query_name: impl Into<String>,
6342        handler: F,
6343    ) where
6344        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6345        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6346    {
6347        self.queries
6348            .entry(workflow_type.into())
6349            .or_default()
6350            .insert(
6351                query_name.into(),
6352                RegisteredQuery::Snapshot(Arc::new(move |ctx, args| Box::pin(handler(ctx, args)))),
6353            );
6354    }
6355
6356    /// Register a named query against deterministically replayed instance state.
6357    ///
6358    /// The workflow type must use [`Worker::register_replayed_workflow`] with the
6359    /// same state type `S`. The handler receives an immutable, detached state
6360    /// clone, so successful and failed queries cannot affect workflow execution
6361    /// or the state reconstructed by a later query.
6362    pub fn register_replayed_query<S, F, Fut>(
6363        &mut self,
6364        workflow_type: impl Into<String>,
6365        query_name: impl Into<String>,
6366        handler: F,
6367    ) where
6368        S: Clone + Send + Sync + 'static,
6369        F: Fn(QueryContext, Arc<S>, Value) -> Fut + Send + Sync + 'static,
6370        Fut: Future<Output = Result<Value>> + Send + 'static,
6371    {
6372        let handler = Arc::new(handler);
6373        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6374            let state = state.downcast::<S>().map_err(|_| {
6375                "registered query state type does not match the replayed workflow state".to_string()
6376            })?;
6377            let handler = Arc::clone(&handler);
6378            Ok(Box::pin(async move {
6379                let result = handler(ctx, state, args.into_json()?).await?;
6380                AvroValue::from_serialize(&result)
6381            }))
6382        });
6383
6384        self.queries
6385            .entry(workflow_type.into())
6386            .or_default()
6387            .insert(
6388                query_name.into(),
6389                RegisteredQuery::Replayed {
6390                    state_type: TypeId::of::<S>(),
6391                    handler: erased_handler,
6392                },
6393            );
6394    }
6395
6396    /// Register a replayed-state query on the lossless fixed Avro Value surface.
6397    pub fn register_replayed_query_avro_value<S, F, Fut>(
6398        &mut self,
6399        workflow_type: impl Into<String>,
6400        query_name: impl Into<String>,
6401        handler: F,
6402    ) where
6403        S: Clone + Send + Sync + 'static,
6404        F: Fn(QueryContext, Arc<S>, AvroValue) -> Fut + Send + Sync + 'static,
6405        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6406    {
6407        let handler = Arc::new(handler);
6408        let erased_handler: ReplayedQueryHandler = Arc::new(move |ctx, state, args| {
6409            let state = state.downcast::<S>().map_err(|_| {
6410                "registered query state type does not match the replayed workflow state".to_string()
6411            })?;
6412            Ok(Box::pin(handler(ctx, state, args)))
6413        });
6414
6415        self.queries
6416            .entry(workflow_type.into())
6417            .or_default()
6418            .insert(
6419                query_name.into(),
6420                RegisteredQuery::Replayed {
6421                    state_type: TypeId::of::<S>(),
6422                    handler: erased_handler,
6423                },
6424            );
6425    }
6426
6427    /// Register a synchronous workflow update handler.
6428    pub fn register_update<F, Fut>(
6429        &mut self,
6430        workflow_type: impl Into<String>,
6431        update_name: impl Into<String>,
6432        handler: F,
6433    ) where
6434        F: Fn(QueryContext, Value) -> Fut + Send + Sync + 'static,
6435        Fut: Future<Output = Result<Value>> + Send + 'static,
6436    {
6437        let handler = Arc::new(handler);
6438        self.updates
6439            .entry(workflow_type.into())
6440            .or_default()
6441            .insert(
6442                update_name.into(),
6443                Arc::new(move |ctx, args| {
6444                    let handler = Arc::clone(&handler);
6445                    Box::pin(async move {
6446                        let result = handler(ctx, args.into_json()?).await?;
6447                        AvroValue::from_serialize(&result)
6448                    })
6449                }),
6450            );
6451    }
6452
6453    /// Register an update handler on the lossless fixed Avro Value surface.
6454    pub fn register_update_avro_value<F, Fut>(
6455        &mut self,
6456        workflow_type: impl Into<String>,
6457        update_name: impl Into<String>,
6458        handler: F,
6459    ) where
6460        F: Fn(QueryContext, AvroValue) -> Fut + Send + Sync + 'static,
6461        Fut: Future<Output = Result<AvroValue>> + Send + 'static,
6462    {
6463        self.updates
6464            .entry(workflow_type.into())
6465            .or_default()
6466            .insert(
6467                update_name.into(),
6468                Arc::new(move |ctx, args| Box::pin(handler(ctx, args))),
6469            );
6470    }
6471
6472    pub async fn register(&self) -> Result<RegisterWorkerResponse> {
6473        let mut command_contracts = serde_json::Map::new();
6474        for workflow_type in self.workflows.keys() {
6475            let mut queries = self
6476                .queries
6477                .get(workflow_type)
6478                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6479                .unwrap_or_default();
6480            queries.sort();
6481            let mut updates = self
6482                .updates
6483                .get(workflow_type)
6484                .map(|handlers| handlers.keys().cloned().collect::<Vec<_>>())
6485                .unwrap_or_default();
6486            updates.sort();
6487            command_contracts.insert(
6488                workflow_type.clone(),
6489                json!({
6490                    "queries": queries,
6491                    "query_contracts": [],
6492                    "signals": [],
6493                    "signal_contracts": [],
6494                    "updates": updates,
6495                    "update_contracts": [],
6496                    "update_validators": [],
6497                }),
6498            );
6499        }
6500
6501        self.client
6502            .register_worker_with_command_contracts(
6503                &self.worker_id,
6504                &self.task_queue,
6505                self.workflows.keys().cloned().collect(),
6506                self.activities.keys().cloned().collect(),
6507                self.max_concurrent_workflow_tasks,
6508                self.max_concurrent_activity_tasks,
6509                [
6510                    Some(CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY.to_string()),
6511                    Some(DURABLE_SELECTION_CAPABILITY.to_string()),
6512                    Some(MEMO_UPSERTS_CAPABILITY.to_string()),
6513                    Some(TYPED_SEARCH_ATTRIBUTES_CAPABILITY.to_string()),
6514                    (!self.queries.is_empty()).then(|| QUERY_TASKS_CAPABILITY.to_string()),
6515                    (!self.updates.is_empty()).then(|| WORKFLOW_UPDATES_CAPABILITY.to_string()),
6516                    worker_protocol_supports_message_streams(WORKER_PROTOCOL_VERSION)
6517                        .then(|| MESSAGE_STREAMS_CAPABILITY.to_string()),
6518                ]
6519                .into_iter()
6520                .flatten()
6521                .collect(),
6522                Value::Object(command_contracts),
6523            )
6524            .await
6525    }
6526
6527    /// Run until shutdown or a terminal worker error occurs.
6528    ///
6529    /// Empty long-poll expirations do not stop the worker. Retryable poll and
6530    /// heartbeat failures use [`WorkerRetryPolicy`] independently, while
6531    /// authentication, protocol, and other non-retryable failures are returned.
6532    pub async fn run(&self) -> Result<()> {
6533        self.run_until(std::future::pending::<()>()).await
6534    }
6535
6536    /// Run until `shutdown` resolves or a terminal worker error occurs.
6537    ///
6538    /// This has the same liveness and terminal-error contract as [`Worker::run`].
6539    pub async fn run_until<F>(&self, shutdown: F) -> Result<()>
6540    where
6541        F: Future<Output = ()>,
6542    {
6543        let stop = Arc::new(AtomicBool::new(false));
6544        let _stop_on_drop = StopWorkerOnDrop(Arc::clone(&stop));
6545        let worker = self.with_storage_admission(Arc::clone(&stop));
6546        let run = worker.run_with_storage_admission(Arc::clone(&stop));
6547        tokio::pin!(run);
6548        tokio::pin!(shutdown);
6549        tokio::select! {
6550            result = &mut run => result,
6551            _ = &mut shutdown => {
6552                stop.store(true, Ordering::SeqCst);
6553                run.await
6554            }
6555        }
6556    }
6557
6558    fn with_storage_admission(&self, stop: Arc<AtomicBool>) -> Self {
6559        let mut worker = self.clone();
6560        worker.client.worker_storage_admission = Some(WorkerStorageAdmission {
6561            policy: self.retry_policy,
6562            stop,
6563        });
6564        worker
6565    }
6566
6567    async fn run_with_storage_admission(&self, stop: Arc<AtomicBool>) -> Result<()> {
6568        let registration = self.register().await?;
6569        if !registration.registered {
6570            return Err(Error::WorkerLoop(format!(
6571                "worker registration for {:?} was not accepted",
6572                self.worker_id
6573            )));
6574        }
6575        let registered_worker_id = registration.worker_id.clone();
6576        let primary = self.run_registered_until(stop, registration).await;
6577        let deregistration = self
6578            .client
6579            .deregister_worker_registration(&registered_worker_id)
6580            .await;
6581
6582        match (primary, deregistration) {
6583            (Ok(()), Ok(_)) => Ok(()),
6584            (Ok(()), Err(deregistration)) => Err(deregistration),
6585            (Err(primary), Ok(_)) => Err(primary),
6586            (Err(primary), Err(deregistration)) => Err(Error::WorkerShutdown {
6587                primary: Box::new(primary),
6588                deregistration: Box::new(deregistration),
6589            }),
6590        }
6591    }
6592
6593    async fn run_registered_until(
6594        &self,
6595        stop: Arc<AtomicBool>,
6596        registration: RegisterWorkerResponse,
6597    ) -> Result<()> {
6598        let heartbeat_interval = Duration::from_secs(
6599            registration
6600                .heartbeat_interval_seconds
6601                .unwrap_or(self.heartbeat_interval.as_secs().max(1)),
6602        );
6603        // The first heartbeat is immediate. Subsequent heartbeats are scheduled
6604        // from the completion of the preceding attempt, including its bounded
6605        // retries. A fixed-epoch interval can leave an already-due tick queued
6606        // while an acknowledgement is slow, producing a catch-up heartbeat as
6607        // soon as that request completes.
6608        let heartbeat = tokio::time::sleep(Duration::ZERO);
6609        tokio::pin!(heartbeat);
6610        // Poll responses may already have leased server-side work by the time
6611        // they become ready, so each poller owns its responses through
6612        // completion or failure instead of racing raw polls in this select.
6613        let mut workflow_poller = (!self.workflows.is_empty()).then(|| {
6614            let worker = self.clone();
6615            let stop = Arc::clone(&stop);
6616            tokio::spawn(async move { worker.poll_workflows_until_stopped(stop).await })
6617        });
6618        let mut activity_poller = (!self.activities.is_empty()).then(|| {
6619            let worker = self.clone();
6620            let stop = Arc::clone(&stop);
6621            tokio::spawn(async move { worker.poll_activities_until_stopped(stop).await })
6622        });
6623        let mut query_poller = (!self.queries.is_empty()).then(|| {
6624            let worker = self.clone();
6625            let stop = Arc::clone(&stop);
6626            tokio::spawn(async move { worker.poll_queries_until_stopped(stop).await })
6627        });
6628
6629        loop {
6630            tokio::select! {
6631                _ = wait_for_worker_stop(&stop) => {
6632                    stop.store(true, Ordering::SeqCst);
6633                    break;
6634                }
6635                _ = &mut heartbeat => {
6636                    let result = self.retry_worker_operation(|| {
6637                        self.client.heartbeat_worker(
6638                            &self.worker_id,
6639                            self.max_concurrent_workflow_tasks,
6640                            self.max_concurrent_activity_tasks,
6641                        )
6642                    }).await;
6643                    heartbeat
6644                        .as_mut()
6645                        .reset(tokio::time::Instant::now() + heartbeat_interval);
6646                    match result {
6647                        Ok(acknowledgement) => {
6648                            if let Some(observer) = &self.heartbeat_observer {
6649                                observer(&WorkerHeartbeatObservation {
6650                                    worker_id: self.worker_id.clone(),
6651                                    task_queue: self.task_queue.clone(),
6652                                    acknowledged_at_unix_millis: SystemTime::now()
6653                                        .duration_since(UNIX_EPOCH)
6654                                        .unwrap_or_default()
6655                                        .as_millis()
6656                                        .min(u64::MAX as u128)
6657                                        as u64,
6658                                    acknowledgement,
6659                                });
6660                            }
6661                        }
6662                        Err(error) => {
6663                            stop.store(true, Ordering::SeqCst);
6664                            join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await?;
6665                            return Err(error);
6666                        }
6667                    }
6668                }
6669                result = OptionFuture::from(workflow_poller.as_mut()), if workflow_poller.is_some() => {
6670                    workflow_poller = None;
6671                    let stopped_by_server = stop.load(Ordering::SeqCst);
6672                    stop.store(true, Ordering::SeqCst);
6673                    let poller_result = optional_poller_result("workflow", result);
6674                    let join_result =
6675                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6676                    poller_result?;
6677                    join_result?;
6678                    if stopped_by_server {
6679                        return Ok(());
6680                    }
6681                    return Err(Error::WorkerLoop(
6682                        "workflow poller stopped unexpectedly".to_string(),
6683                    ));
6684                }
6685                result = OptionFuture::from(activity_poller.as_mut()), if activity_poller.is_some() => {
6686                    activity_poller = None;
6687                    let stopped_by_server = stop.load(Ordering::SeqCst);
6688                    stop.store(true, Ordering::SeqCst);
6689                    let poller_result = optional_poller_result("activity", result);
6690                    let join_result =
6691                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6692                    poller_result?;
6693                    join_result?;
6694                    if stopped_by_server {
6695                        return Ok(());
6696                    }
6697                    return Err(Error::WorkerLoop(
6698                        "activity poller stopped unexpectedly".to_string(),
6699                    ));
6700                }
6701                result = OptionFuture::from(query_poller.as_mut()), if query_poller.is_some() => {
6702                    query_poller = None;
6703                    let stopped_by_server = stop.load(Ordering::SeqCst);
6704                    stop.store(true, Ordering::SeqCst);
6705                    let poller_result = optional_poller_result("query", result);
6706                    let join_result =
6707                        join_pollers(workflow_poller.take(), activity_poller.take(), query_poller.take()).await;
6708                    poller_result?;
6709                    join_result?;
6710                    if stopped_by_server {
6711                        return Ok(());
6712                    }
6713                    return Err(Error::WorkerLoop(
6714                        "query poller stopped unexpectedly".to_string(),
6715                    ));
6716                }
6717            }
6718        }
6719
6720        join_pollers(
6721            workflow_poller.take(),
6722            activity_poller.take(),
6723            query_poller.take(),
6724        )
6725        .await
6726    }
6727
6728    /// Poll and settle at most one task from each enabled task family.
6729    ///
6730    /// A workflow may reach its server-enforced run deadline while this worker
6731    /// holds a task. When the completion endpoint authoritatively rejects that
6732    /// selected task and run with `recorded=false`, `reason=run_timed_out`, and
6733    /// terminal `run_status=failed`, the workflow tick is considered settled:
6734    /// the late command was not recorded and cannot replace the terminal run.
6735    /// Every other completion rejection remains an error. This worker-level
6736    /// race handling is distinct from [`WorkflowResultOptions::timeout`], which
6737    /// only bounds how long a client waits for a result.
6738    ///
6739    /// Direct callers of [`Client::complete_workflow_task`] continue to receive
6740    /// the original [`Error::Http`] status and response body.
6741    pub async fn run_once(&self) -> Result<usize> {
6742        let worker = self.with_storage_admission(Arc::new(AtomicBool::new(false)));
6743        let mut handled = 0;
6744        match worker.poll_workflow_once().await? {
6745            ManagedPollOutcome::Handled => handled += 1,
6746            ManagedPollOutcome::Stop => return Ok(handled),
6747            ManagedPollOutcome::Idle => {}
6748        }
6749        match worker.poll_activity_once().await? {
6750            ManagedPollOutcome::Handled => handled += 1,
6751            ManagedPollOutcome::Stop => return Ok(handled),
6752            ManagedPollOutcome::Idle => {}
6753        }
6754        if !self.queries.is_empty() {
6755            match worker.poll_query_once().await? {
6756                ManagedPollOutcome::Handled => handled += 1,
6757                ManagedPollOutcome::Stop => return Ok(handled),
6758                ManagedPollOutcome::Idle => {}
6759            }
6760        }
6761        Ok(handled)
6762    }
6763
6764    async fn poll_workflow_once(&self) -> Result<ManagedPollOutcome> {
6765        let poll_request_id = unique_request_id("rust-workflow-poll");
6766        let response = self
6767            .retry_worker_operation(|| {
6768                self.client.poll_workflow_task_response_with_request_id(
6769                    &self.worker_id,
6770                    &self.task_queue,
6771                    self.poll_timeout,
6772                    &poll_request_id,
6773                    0,
6774                )
6775            })
6776            .await;
6777        let Some(response) = self.settle_worker_poll_response(response).await? else {
6778            return Ok(ManagedPollOutcome::Idle);
6779        };
6780        if response.outcome().should_stop() {
6781            return Ok(ManagedPollOutcome::Stop);
6782        }
6783        let memo_updates_supported =
6784            runtime_supports_workflow_memo_updates(response.server_capabilities.as_ref());
6785        let Some(task) = response.task else {
6786            return Ok(ManagedPollOutcome::Idle);
6787        };
6788
6789        let task_id = task.task_id.clone();
6790        let attempt = task.workflow_task_attempt;
6791        let run_id = task.run_id.clone();
6792        let lease_owner = task
6793            .lease_owner
6794            .clone()
6795            .unwrap_or_else(|| self.worker_id.clone());
6796
6797        match self.execute_workflow_task_decision(task) {
6798            Ok(decision)
6799                if commands_use_workflow_memo_updates(&decision.commands)
6800                    && !memo_updates_supported =>
6801            {
6802                self.client
6803                    .fail_workflow_task(
6804                        &task_id,
6805                        &lease_owner,
6806                        attempt,
6807                        Error::WorkflowMemoUpdatesUnavailable.to_string(),
6808                    )
6809                    .await?;
6810            }
6811            Ok(decision) if decision.commands.is_empty() => {
6812                // A replay can consume a recorded pending durable command
6813                // without producing a new command. The standalone protocol
6814                // acknowledges that state through the typed waiting outcome;
6815                // an empty completion is rejected by servers that require at
6816                // least one executable command.
6817                self.client
6818                    .fail_workflow_task_with_type(
6819                        &task_id,
6820                        &lease_owner,
6821                        attempt,
6822                        WORKFLOW_TASK_WAITING_FOR_HISTORY_MESSAGE,
6823                        WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE,
6824                    )
6825                    .await?;
6826            }
6827            Ok(decision) => {
6828                let completion = self
6829                    .client
6830                    .complete_workflow_task_with_message_streams(
6831                        &task_id,
6832                        &lease_owner,
6833                        attempt,
6834                        decision.commands,
6835                        decision.message_stream_cursors,
6836                        decision.message_stream_waits,
6837                    )
6838                    .await;
6839                if let Err(error) = completion {
6840                    if !workflow_task_completion_is_terminal_timeout(
6841                        &error,
6842                        &task_id,
6843                        attempt,
6844                        run_id.as_deref(),
6845                    ) {
6846                        return Err(error);
6847                    }
6848                }
6849            }
6850            Err(error) => {
6851                self.client
6852                    .fail_workflow_task(&task_id, &lease_owner, attempt, error.to_string())
6853                    .await?;
6854            }
6855        }
6856
6857        Ok(ManagedPollOutcome::Handled)
6858    }
6859
6860    async fn poll_workflows_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6861        while !stop.load(Ordering::SeqCst) {
6862            if self.poll_workflow_once().await? == ManagedPollOutcome::Stop {
6863                stop.store(true, Ordering::SeqCst);
6864                break;
6865            }
6866        }
6867
6868        Ok(())
6869    }
6870
6871    async fn poll_activity_once(&self) -> Result<ManagedPollOutcome> {
6872        let poll_request_id = unique_request_id("rust-activity-poll");
6873        let response = self
6874            .retry_worker_operation(|| {
6875                self.client.poll_activity_task_response_with_request_id(
6876                    &self.worker_id,
6877                    &self.task_queue,
6878                    self.poll_timeout,
6879                    &poll_request_id,
6880                    0,
6881                )
6882            })
6883            .await;
6884        let Some(response) = self.settle_worker_poll_response(response).await? else {
6885            return Ok(ManagedPollOutcome::Idle);
6886        };
6887        if response.outcome().should_stop() {
6888            return Ok(ManagedPollOutcome::Stop);
6889        }
6890        let Some(task) = response.task else {
6891            return Ok(ManagedPollOutcome::Idle);
6892        };
6893
6894        let task_id = task.task_id.clone();
6895        let attempt_id = task
6896            .activity_attempt_id
6897            .clone()
6898            .or(task.attempt_id.clone())
6899            .unwrap_or_default();
6900        let lease_owner = task
6901            .lease_owner
6902            .clone()
6903            .unwrap_or_else(|| self.worker_id.clone());
6904        let codec = task.payload_codec.clone();
6905        let result = self.execute_activity_task(task).await;
6906        match result {
6907            Err(error) if worker_storage_admission_body(&error).is_some() => return Err(error),
6908            Ok(value) => {
6909                let completion = self
6910                    .client
6911                    .complete_activity_task(&task_id, &attempt_id, &lease_owner, value, &codec)
6912                    .await;
6913                if let Err(error) = completion {
6914                    if !activity_task_rejection_is_final(&error) {
6915                        return Err(error);
6916                    }
6917                }
6918            }
6919            Err(error) => {
6920                let failure = self
6921                    .client
6922                    .fail_activity_task(
6923                        &task_id,
6924                        &attempt_id,
6925                        &lease_owner,
6926                        error.to_string(),
6927                        false,
6928                    )
6929                    .await;
6930                if let Err(error) = failure {
6931                    if !activity_task_rejection_is_final(&error) {
6932                        return Err(error);
6933                    }
6934                }
6935            }
6936        }
6937
6938        Ok(ManagedPollOutcome::Handled)
6939    }
6940
6941    async fn poll_activities_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
6942        while !stop.load(Ordering::SeqCst) {
6943            if self.poll_activity_once().await? == ManagedPollOutcome::Stop {
6944                stop.store(true, Ordering::SeqCst);
6945                break;
6946            }
6947        }
6948
6949        Ok(())
6950    }
6951
6952    async fn poll_query_once(&self) -> Result<ManagedPollOutcome> {
6953        let poll_request_id = unique_request_id("rust-query-poll");
6954        let response = self
6955            .retry_worker_operation(|| {
6956                self.client.poll_query_task_response_with_request_id(
6957                    &self.worker_id,
6958                    &self.task_queue,
6959                    self.poll_timeout,
6960                    &poll_request_id,
6961                    0,
6962                )
6963            })
6964            .await;
6965        let Some(response) = self.settle_worker_poll_response(response).await? else {
6966            return Ok(ManagedPollOutcome::Idle);
6967        };
6968        if response.outcome().should_stop() {
6969            return Ok(ManagedPollOutcome::Stop);
6970        }
6971        let Some(task) = response.task else {
6972            return Ok(ManagedPollOutcome::Idle);
6973        };
6974
6975        let query_task_id = task.query_task_id.clone();
6976        let attempt = task.query_task_attempt;
6977        let lease_owner = task
6978            .lease_owner
6979            .clone()
6980            .unwrap_or_else(|| self.worker_id.clone());
6981        let codec = task.payload_codec.clone();
6982
6983        match self.execute_query_task(task).await {
6984            Ok(value) => {
6985                let result_envelope = match encode_typed_envelope(&value, &codec) {
6986                    Ok(result_envelope) => result_envelope,
6987                    Err(error) => {
6988                        let failure = self
6989                            .client
6990                            .fail_query_task(
6991                                &query_task_id,
6992                                &lease_owner,
6993                                attempt,
6994                                error.to_string(),
6995                                "query_result_encode_failed",
6996                                "QueryResultEncodeFailed",
6997                            )
6998                            .await;
6999                        if let Err(error) = failure {
7000                            if !query_task_rejection_is_final(&error) {
7001                                return Err(error);
7002                            }
7003                        }
7004                        return Ok(ManagedPollOutcome::Handled);
7005                    }
7006                };
7007
7008                if let Err(error) = self
7009                    .client
7010                    .complete_query_task_with_envelope(
7011                        &query_task_id,
7012                        &lease_owner,
7013                        attempt,
7014                        value.clone().into_json()?,
7015                        result_envelope,
7016                    )
7017                    .await
7018                {
7019                    if !query_task_rejection_is_final(&error) {
7020                        return Err(error);
7021                    }
7022                }
7023            }
7024            Err(failure) => {
7025                let result = self
7026                    .client
7027                    .fail_query_task(
7028                        &query_task_id,
7029                        &lease_owner,
7030                        attempt,
7031                        failure.message,
7032                        failure.reason,
7033                        failure.failure_type,
7034                    )
7035                    .await;
7036                if let Err(error) = result {
7037                    if !query_task_rejection_is_final(&error) {
7038                        return Err(error);
7039                    }
7040                }
7041            }
7042        }
7043
7044        Ok(ManagedPollOutcome::Handled)
7045    }
7046
7047    async fn poll_queries_until_stopped(self, stop: Arc<AtomicBool>) -> Result<()> {
7048        while !stop.load(Ordering::SeqCst) {
7049            if self.poll_query_once().await? == ManagedPollOutcome::Stop {
7050                stop.store(true, Ordering::SeqCst);
7051                break;
7052            }
7053        }
7054
7055        Ok(())
7056    }
7057
7058    async fn retry_worker_operation<T, F, Fut>(&self, mut operation: F) -> Result<T>
7059    where
7060        F: FnMut() -> Fut,
7061        Fut: Future<Output = Result<T>>,
7062    {
7063        let mut retries = 0;
7064
7065        loop {
7066            match operation().await {
7067                Err(error)
7068                    if worker_operation_is_retryable(&error)
7069                        && retries < self.retry_policy.max_retries =>
7070                {
7071                    retries += 1;
7072                    tokio::time::sleep(worker_retry_delay(self.retry_policy, retries)).await;
7073                }
7074                result => return result,
7075            }
7076        }
7077    }
7078
7079    async fn settle_worker_poll_response<T>(&self, response: Result<T>) -> Result<Option<T>> {
7080        match response {
7081            Ok(response) => Ok(Some(response)),
7082            Err(error) => {
7083                let Some(advertised_delay) = worker_poll_capacity_retry_after(&error) else {
7084                    return Err(error);
7085                };
7086                let minimum_delay = self
7087                    .retry_policy
7088                    .initial_backoff
7089                    .max(Duration::from_millis(1));
7090                let maximum_delay = self.retry_policy.max_backoff.max(minimum_delay);
7091                tokio::time::sleep(advertised_delay.max(minimum_delay).min(maximum_delay)).await;
7092                Ok(None)
7093            }
7094        }
7095    }
7096
7097    async fn execute_query_task(
7098        &self,
7099        mut task: QueryTask,
7100    ) -> std::result::Result<AvroValue, QueryTaskExecutionFailure> {
7101        validate_query_task_payloads(&task).map_err(|error| {
7102            QueryTaskExecutionFailure::new(
7103                "query_payload_decode_failed",
7104                error.to_string(),
7105                "QueryPayloadDecodeFailed",
7106            )
7107        })?;
7108
7109        if !self.workflows.contains_key(&task.workflow_type) {
7110            return Err(QueryTaskExecutionFailure::new(
7111                "query_workflow_type_not_registered",
7112                format!("no workflow registered for type {:?}", task.workflow_type),
7113                "WorkflowTypeNotRegistered",
7114            ));
7115        }
7116
7117        let Some(handlers) = self.queries.get(&task.workflow_type) else {
7118            return Err(QueryTaskExecutionFailure::new(
7119                "query_handler_unavailable",
7120                format!(
7121                    "query handlers are unavailable for workflow type {:?}",
7122                    task.workflow_type
7123                ),
7124                "QueryHandlerUnavailable",
7125            ));
7126        };
7127        let Some(query) = handlers.get(&task.query_name) else {
7128            return Err(QueryTaskExecutionFailure::new(
7129                "rejected_unknown_query",
7130                format!("unknown query {:?}", task.query_name),
7131                "QueryFailed",
7132            ));
7133        };
7134
7135        let args = decode_task_avro_arguments(task.query_arguments.as_ref(), &task.payload_codec)
7136            .map_err(|error| {
7137            QueryTaskExecutionFailure::new(
7138                "query_payload_decode_failed",
7139                format!("cannot decode query arguments: {error}"),
7140                "QueryPayloadDecodeFailed",
7141            )
7142        })?;
7143        let workflow_input_typed =
7144            decode_task_avro_arguments(task.workflow_arguments.as_ref(), &task.payload_codec)
7145                .map_err(|error| {
7146                    QueryTaskExecutionFailure::new(
7147                        "query_workflow_state_unavailable",
7148                        format!("cannot decode workflow start input: {error}"),
7149                        "QueryWorkflowStateUnavailable",
7150                    )
7151                })?;
7152        let workflow_input = workflow_input_typed.clone().into_json().map_err(|error| {
7153            QueryTaskExecutionFailure::new(
7154                "query_workflow_state_unavailable",
7155                format!("cannot project workflow start input: {error}"),
7156                "QueryWorkflowStateUnavailable",
7157            )
7158        })?;
7159        hydrate_query_history_from_export(&mut task).map_err(|error| {
7160            QueryTaskExecutionFailure::new(
7161                "query_workflow_state_unavailable",
7162                format!("cannot restore query history snapshot: {error}"),
7163                "QueryWorkflowStateUnavailable",
7164            )
7165        })?;
7166        enrich_query_history_from_export(&mut task).map_err(|error| {
7167            QueryTaskExecutionFailure::new(
7168                "query_workflow_state_unavailable",
7169                format!("cannot restore compact query history payloads: {error}"),
7170                "QueryWorkflowStateUnavailable",
7171            )
7172        })?;
7173        let signal_events = query_signal_events(&task).map_err(|error| {
7174            QueryTaskExecutionFailure::new(
7175                "query_workflow_state_unavailable",
7176                format!("cannot decode committed workflow signals: {error}"),
7177                "QueryWorkflowStateUnavailable",
7178            )
7179        })?;
7180        let history_events = Arc::new(std::mem::take(&mut task.history_events));
7181        let context = QueryContext {
7182            workflow_id: task.workflow_id,
7183            run_id: task.run_id,
7184            workflow_type: task.workflow_type.clone(),
7185            run_status: task.run_status,
7186            workflow_input,
7187            workflow_input_avro_value: workflow_input_typed.clone(),
7188            history_events: Arc::clone(&history_events),
7189            signal_events: Arc::new(signal_events),
7190        };
7191
7192        let future = match query {
7193            RegisteredQuery::Snapshot(handler) => handler(context, args),
7194            RegisteredQuery::Replayed {
7195                state_type,
7196                handler,
7197            } => {
7198                let workflow = self
7199                    .workflows
7200                    .get(&task.workflow_type)
7201                    .expect("workflow registration was checked above");
7202                if workflow.state_type != Some(*state_type) {
7203                    return Err(QueryTaskExecutionFailure::new(
7204                        "query_workflow_state_unavailable",
7205                        "replayed query state type does not match its workflow registration",
7206                        "QueryWorkflowStateUnavailable",
7207                    ));
7208                }
7209                let replay = workflow.replay.as_ref().ok_or_else(|| {
7210                    QueryTaskExecutionFailure::new(
7211                        "query_workflow_state_unavailable",
7212                        format!(
7213                            "workflow type {:?} is not registered for instance-state replay",
7214                            task.workflow_type
7215                        ),
7216                        "QueryWorkflowStateUnavailable",
7217                    )
7218                })?;
7219                let workflow_state = Arc::new(Mutex::new(
7220                    WorkflowState::new_with_identity(
7221                        history_events.as_ref().clone(),
7222                        context.workflow_id.clone(),
7223                        context.run_id.clone(),
7224                        self.task_queue.clone(),
7225                        task.payload_codec,
7226                        None,
7227                    )
7228                    .map_err(|error| {
7229                        QueryTaskExecutionFailure::new(
7230                            "query_workflow_state_unavailable",
7231                            format!("workflow replay failed before query: {error}"),
7232                            "QueryWorkflowStateUnavailable",
7233                        )
7234                    })?,
7235                ));
7236                let workflow_context = WorkflowContext {
7237                    state: workflow_state,
7238                };
7239                let mut invocation = replay(workflow_context.clone(), workflow_input_typed.clone());
7240                let mut cx = TaskContext::from_waker(noop_waker_ref());
7241                match invocation.future.as_mut().poll(&mut cx) {
7242                    Poll::Ready(Ok(_)) => {
7243                        workflow_context
7244                            .ensure_history_consumed()
7245                            .map_err(|error| {
7246                                QueryTaskExecutionFailure::new(
7247                                    "query_workflow_state_unavailable",
7248                                    format!("workflow replay failed before query: {error}"),
7249                                    "QueryWorkflowStateUnavailable",
7250                                )
7251                            })?;
7252                    }
7253                    Poll::Ready(Err(error)) => {
7254                        return Err(QueryTaskExecutionFailure::new(
7255                            "query_workflow_state_unavailable",
7256                            format!("workflow replay failed before query: {error}"),
7257                            "QueryWorkflowStateUnavailable",
7258                        ));
7259                    }
7260                    Poll::Pending => {
7261                        let commands = workflow_context.take_commands().map_err(|error| {
7262                            QueryTaskExecutionFailure::new(
7263                                "query_workflow_state_unavailable",
7264                                format!("workflow replay failed before query: {error}"),
7265                                "QueryWorkflowStateUnavailable",
7266                            )
7267                        })?;
7268                        if commands.is_empty()
7269                            && !workflow_context
7270                                .matched_recorded_pending()
7271                                .map_err(|error| {
7272                                    QueryTaskExecutionFailure::new(
7273                                        "query_workflow_state_unavailable",
7274                                        format!("workflow replay failed before query: {error}"),
7275                                        "QueryWorkflowStateUnavailable",
7276                                    )
7277                                })?
7278                        {
7279                            return Err(QueryTaskExecutionFailure::new(
7280                                "query_workflow_state_unavailable",
7281                                "workflow replay yielded without a durable command",
7282                                "QueryWorkflowStateUnavailable",
7283                            ));
7284                        }
7285                    }
7286                }
7287                let state = (invocation.snapshot)().map_err(|error| {
7288                    QueryTaskExecutionFailure::new(
7289                        "query_workflow_state_unavailable",
7290                        format!("cannot snapshot replayed workflow state: {error}"),
7291                        "QueryWorkflowStateUnavailable",
7292                    )
7293                })?;
7294                handler(context, state, args).map_err(|message| {
7295                    QueryTaskExecutionFailure::new(
7296                        "query_workflow_state_unavailable",
7297                        message,
7298                        "QueryWorkflowStateUnavailable",
7299                    )
7300                })?
7301            }
7302        };
7303
7304        future.await.map_err(|error| {
7305            QueryTaskExecutionFailure::new("query_rejected", error.to_string(), "QueryFailed")
7306        })
7307    }
7308
7309    #[cfg(test)]
7310    fn execute_workflow_task(&self, task: WorkflowTask) -> Result<Vec<Value>> {
7311        Ok(self.execute_workflow_task_decision(task)?.commands)
7312    }
7313
7314    fn execute_workflow_task_decision(&self, task: WorkflowTask) -> Result<WorkflowTaskDecision> {
7315        validate_workflow_task_payloads(&task)?;
7316
7317        if let Some(update_id) = task
7318            .workflow_update_id
7319            .as_deref()
7320            .filter(|update_id| !update_id.is_empty())
7321        {
7322            return self
7323                .execute_update_task(&task, update_id)
7324                .map(WorkflowTaskDecision::without_message_streams);
7325        }
7326
7327        let workflow = self
7328            .workflows
7329            .get(&task.workflow_type)
7330            .ok_or_else(|| Error::WorkflowNotRegistered(task.workflow_type.clone()))?;
7331        let input = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7332        let resume_signal = decode_resume_signal(&task)?;
7333        let history_budget = WorkflowHistoryBudget {
7334            event_count: task
7335                .total_history_events
7336                .unwrap_or_else(|| u64::try_from(task.history_events.len()).unwrap_or(u64::MAX)),
7337            size_bytes: task.history_size_bytes,
7338            continue_as_new_recommended: task.continue_as_new_recommended.unwrap_or(false),
7339            pressure: task.history_budget_pressure.clone(),
7340        };
7341        let workflow_command_identity = task
7342            .workflow_command_id
7343            .clone()
7344            .filter(|identity| !identity.is_empty())
7345            .unwrap_or_default();
7346        let mut workflow_state = WorkflowState::new_with_identity(
7347            task.history_events,
7348            task.workflow_id,
7349            task.run_id,
7350            self.task_queue.clone(),
7351            task.payload_codec.clone(),
7352            resume_signal,
7353        )?;
7354        workflow_state.history_budget = history_budget;
7355        workflow_state.workflow_command_identity = workflow_command_identity;
7356        workflow_state.cancel_requested = task.cancel_requested;
7357        let state = Arc::new(Mutex::new(workflow_state));
7358        let ctx = WorkflowContext { state };
7359        let mut future = (workflow.execute)(ctx.clone(), input);
7360        let mut cx = TaskContext::from_waker(noop_waker_ref());
7361
7362        match future.as_mut().poll(&mut cx) {
7363            Poll::Ready(Ok(result)) => {
7364                ctx.ensure_history_consumed()?;
7365                let result = encode_typed_envelope(&result, &task.payload_codec)?;
7366                let mut commands = ctx.take_commands()?;
7367                commands.push(json!({
7368                    "type": "complete_workflow",
7369                    "result": result
7370                }));
7371                self.message_stream_decision(&ctx, commands)
7372            }
7373            Poll::Ready(Err(error)) => {
7374                if let Error::ContinueAsNew(request) = error {
7375                    let mut commands = ctx.take_commands()?;
7376                    if let Some(command) = ctx.continue_as_new_command(request)? {
7377                        commands.push(command);
7378                    }
7379                    ctx.ensure_history_consumed()?;
7380                    return self.message_stream_decision(&ctx, commands);
7381                }
7382                if workflow_task_integrity_error(&error) {
7383                    // Replay and protocol failures describe the workflow-task
7384                    // decision itself. Preserve their specific failure reason
7385                    // instead of replacing it with the derivative fact that
7386                    // recorded commands remain unconsumed.
7387                    return Err(error);
7388                }
7389                // A handler error must not hide a committed durable command that
7390                // upgraded workflow code no longer consumes.
7391                ctx.ensure_history_consumed()?;
7392                let mut commands = ctx.take_commands()?;
7393                commands.push(workflow_failure_command(&error));
7394                self.message_stream_decision(&ctx, commands)
7395            }
7396            Poll::Pending => {
7397                let commands = ctx.take_commands()?;
7398                if commands.is_empty() && !ctx.matched_recorded_pending()? {
7399                    Err(Error::WorkflowYieldedWithoutCommand)
7400                } else {
7401                    self.message_stream_decision(&ctx, commands)
7402                }
7403            }
7404        }
7405    }
7406
7407    fn message_stream_decision(
7408        &self,
7409        ctx: &WorkflowContext,
7410        commands: Vec<Value>,
7411    ) -> Result<WorkflowTaskDecision> {
7412        let (message_stream_cursors, message_stream_waits) = ctx.message_stream_metadata()?;
7413        Ok(WorkflowTaskDecision {
7414            commands,
7415            message_stream_cursors,
7416            message_stream_waits,
7417        })
7418    }
7419
7420    fn execute_update_task(&self, task: &WorkflowTask, update_id: &str) -> Result<Vec<Value>> {
7421        if !self.workflows.contains_key(&task.workflow_type) {
7422            return Err(Error::WorkflowNotRegistered(task.workflow_type.clone()));
7423        }
7424
7425        let accepted = task.history_events.iter().rev().find_map(|event| {
7426            (event.event_type == "UpdateAccepted"
7427                && event.payload.get("update_id").and_then(Value::as_str) == Some(update_id))
7428            .then_some(&event.payload)
7429        });
7430        let update_name = accepted
7431            .and_then(|payload| payload.get("update_name"))
7432            .and_then(Value::as_str)
7433            .or(task.update_name.as_deref())
7434            .unwrap_or_default();
7435        let Some(handler) = self
7436            .updates
7437            .get(&task.workflow_type)
7438            .and_then(|handlers| handlers.get(update_name))
7439        else {
7440            return Ok(vec![json!({
7441                "type": "fail_update",
7442                "update_id": update_id,
7443                "message": format!(
7444                    "no update handler is registered for {}.{update_name}",
7445                    task.workflow_type
7446                ),
7447                "exception_type": "UnknownUpdate",
7448                "non_retryable": true,
7449            })]);
7450        };
7451        let arguments = accepted
7452            .and_then(|payload| payload.get("arguments"))
7453            .or(task.arguments.as_ref());
7454        let arguments = decode_task_avro_arguments(arguments, &task.payload_codec)?;
7455        let context = QueryContext {
7456            workflow_id: task.workflow_id.clone(),
7457            run_id: task.run_id.clone(),
7458            workflow_type: task.workflow_type.clone(),
7459            run_status: Some("running".to_string()),
7460            workflow_input: Value::Null,
7461            workflow_input_avro_value: AvroValue::Null,
7462            history_events: Arc::new(task.history_events.clone()),
7463            signal_events: Arc::new(Vec::new()),
7464        };
7465        let mut future = handler(context, arguments);
7466        let mut cx = TaskContext::from_waker(noop_waker_ref());
7467
7468        match future.as_mut().poll(&mut cx) {
7469            Poll::Ready(Ok(result)) => Ok(vec![json!({
7470                "type": "complete_update",
7471                "update_id": update_id,
7472                "result": encode_typed_envelope(&result, &task.payload_codec)?,
7473            })]),
7474            Poll::Ready(Err(error)) => Ok(vec![json!({
7475                "type": "fail_update",
7476                "update_id": update_id,
7477                "message": error.to_string(),
7478                "exception_type": "UpdateFailed",
7479                "non_retryable": true,
7480            })]),
7481            Poll::Pending => Err(Error::WorkflowYieldedWithoutCommand),
7482        }
7483    }
7484
7485    async fn execute_activity_task(&self, task: ActivityTask) -> Result<AvroValue> {
7486        validate_activity_task_payloads(&task)?;
7487
7488        let handler = self
7489            .activities
7490            .get(&task.activity_type)
7491            .ok_or_else(|| Error::ActivityNotRegistered(task.activity_type.clone()))?;
7492        let args = decode_task_avro_arguments(task.arguments.as_ref(), &task.payload_codec)?;
7493        let attempt_id = task
7494            .activity_attempt_id
7495            .clone()
7496            .or(task.attempt_id.clone())
7497            .unwrap_or_default();
7498        let lease_owner = task
7499            .lease_owner
7500            .clone()
7501            .unwrap_or_else(|| self.worker_id.clone());
7502        let ctx = ActivityContext {
7503            client: self.client.clone(),
7504            task_id: task.task_id,
7505            activity_attempt_id: attempt_id,
7506            lease_owner,
7507            activity_type: task.activity_type,
7508            attempt_number: task.attempt_number,
7509            task_queue: self.task_queue.clone(),
7510            worker_id: self.worker_id.clone(),
7511        };
7512
7513        handler(ctx, args).await
7514    }
7515}
7516
7517fn poller_result(
7518    kind: &str,
7519    result: std::result::Result<Result<()>, tokio::task::JoinError>,
7520) -> Result<()> {
7521    match result {
7522        Ok(result) => result,
7523        Err(error) => Err(Error::WorkerLoop(format!(
7524            "{kind} poller join error: {error}"
7525        ))),
7526    }
7527}
7528
7529fn optional_poller_result(
7530    kind: &str,
7531    result: Option<std::result::Result<Result<()>, tokio::task::JoinError>>,
7532) -> Result<()> {
7533    match result {
7534        Some(result) => poller_result(kind, result),
7535        None => Ok(()),
7536    }
7537}
7538
7539async fn join_pollers(
7540    workflow_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7541    activity_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7542    query_poller: Option<tokio::task::JoinHandle<Result<()>>>,
7543) -> Result<()> {
7544    let mut first_error = None;
7545
7546    if let Some(handle) = workflow_poller {
7547        if let Err(error) = poller_result("workflow", handle.await) {
7548            first_error.get_or_insert(error);
7549        }
7550    }
7551
7552    if let Some(handle) = activity_poller {
7553        if let Err(error) = poller_result("activity", handle.await) {
7554            first_error.get_or_insert(error);
7555        }
7556    }
7557
7558    if let Some(handle) = query_poller {
7559        if let Err(error) = poller_result("query", handle.await) {
7560            first_error.get_or_insert(error);
7561        }
7562    }
7563
7564    if let Some(error) = first_error {
7565        Err(error)
7566    } else {
7567        Ok(())
7568    }
7569}
7570
7571fn default_worker_id() -> String {
7572    let millis = SystemTime::now()
7573        .duration_since(UNIX_EPOCH)
7574        .unwrap_or_default()
7575        .as_millis();
7576    format!("rust-worker-{}-{millis}", std::process::id())
7577}
7578
7579fn percent_encode_path_segment(segment: &str) -> String {
7580    const HEX: &[u8; 16] = b"0123456789ABCDEF";
7581    let mut encoded = String::with_capacity(segment.len());
7582
7583    for byte in segment.bytes() {
7584        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
7585            encoded.push(char::from(byte));
7586        } else {
7587            encoded.push('%');
7588            encoded.push(char::from(HEX[(byte >> 4) as usize]));
7589            encoded.push(char::from(HEX[(byte & 0x0f) as usize]));
7590        }
7591    }
7592
7593    encoded
7594}
7595
7596fn unique_request_id(prefix: &str) -> String {
7597    let nanos = SystemTime::now()
7598        .duration_since(UNIX_EPOCH)
7599        .unwrap_or_default()
7600        .as_nanos();
7601    format!("{prefix}-{}-{nanos}", std::process::id())
7602}
7603
7604#[derive(Debug)]
7605struct QueryTaskExecutionFailure {
7606    reason: String,
7607    message: String,
7608    failure_type: String,
7609}
7610
7611impl QueryTaskExecutionFailure {
7612    fn new(
7613        reason: impl Into<String>,
7614        message: impl Into<String>,
7615        failure_type: impl Into<String>,
7616    ) -> Self {
7617        Self {
7618            reason: reason.into(),
7619            message: message.into(),
7620            failure_type: failure_type.into(),
7621        }
7622    }
7623}
7624
7625/// Typed local state owned by one deterministic workflow invocation.
7626///
7627/// Use [`WorkflowInstance::update`] for the same state transitions during
7628/// ordinary execution and replay. A replayed query receives a detached
7629/// immutable `Arc<S>` rather than this mutation-capable handle.
7630#[derive(Clone, Debug)]
7631pub struct WorkflowInstance<S> {
7632    state: Arc<Mutex<S>>,
7633}
7634
7635impl<S> WorkflowInstance<S> {
7636    fn new(state: S) -> Self {
7637        Self {
7638            state: Arc::new(Mutex::new(state)),
7639        }
7640    }
7641
7642    /// Read the current workflow-instance state without changing it.
7643    pub fn read<R>(&self, reader: impl FnOnce(&S) -> R) -> Result<R> {
7644        let state = self
7645            .state
7646            .lock()
7647            .map_err(|_| Error::WorkflowStatePoisoned)?;
7648        Ok(reader(&state))
7649    }
7650
7651    /// Apply one deterministic workflow-instance state transition.
7652    pub fn update<R>(&self, transition: impl FnOnce(&mut S) -> R) -> Result<R> {
7653        let mut state = self
7654            .state
7655            .lock()
7656            .map_err(|_| Error::WorkflowStatePoisoned)?;
7657        Ok(transition(&mut state))
7658    }
7659}
7660
7661impl<S: Clone> WorkflowInstance<S> {
7662    fn snapshot(&self) -> Result<S> {
7663        self.read(Clone::clone)
7664    }
7665}
7666
7667#[derive(Clone, Debug, PartialEq)]
7668pub struct MessageStreamMessage {
7669    pub stream_name: String,
7670    pub message_id: String,
7671    pub position: u64,
7672    pub arguments: Vec<AvroValue>,
7673}
7674
7675#[derive(Clone, Debug)]
7676pub struct MessageStream {
7677    ctx: WorkflowContext,
7678    name: String,
7679}
7680
7681impl MessageStream {
7682    /// Wait for one message, then return a bounded currently-available batch.
7683    pub async fn receive(&self, max_items: usize) -> Result<Vec<MessageStreamMessage>> {
7684        if !(1..=MESSAGE_STREAM_MAX_BATCH).contains(&max_items) {
7685            return Err(Error::Codec(format!(
7686                "message stream max_items must be between 1 and {MESSAGE_STREAM_MAX_BATCH}"
7687            )));
7688        }
7689        loop {
7690            if let Some(batch) = self.ctx.take_message_stream_batch(&self.name, max_items)? {
7691                return Ok(batch);
7692            }
7693
7694            self.ctx.record_message_stream_wait(&self.name)?;
7695            let replay_wait_sequence = self.ctx.next_message_stream_wait_sequence()?;
7696            let arguments = self.ctx.wait_runtime_signal(MESSAGE_STREAM_SIGNAL).await?;
7697            self.ctx.buffer_message_stream_delivery(arguments)?;
7698            if let Some(sequence) = replay_wait_sequence {
7699                self.ctx.buffer_message_stream_history_for_wait(sequence)?;
7700            }
7701        }
7702    }
7703
7704    pub async fn receive_one(&self) -> Result<MessageStreamMessage> {
7705        self.receive(1)
7706            .await?
7707            .into_iter()
7708            .next()
7709            .ok_or_else(|| Error::Codec("message stream resumed without a message".to_string()))
7710    }
7711}
7712
7713#[derive(Clone, Debug)]
7714pub struct WorkflowContext {
7715    state: Arc<Mutex<WorkflowState>>,
7716}
7717
7718fn valid_memo_key(key: &str) -> bool {
7719    let numeric_candidate = key.strip_prefix('-').unwrap_or(key);
7720
7721    !key.is_empty()
7722        && key.len() <= 64
7723        && (numeric_candidate.is_empty()
7724            || !numeric_candidate.bytes().all(|byte| byte.is_ascii_digit()))
7725        && key
7726            .bytes()
7727            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b':' | b'-'))
7728}
7729
7730fn avro_encoded_size(value: &AvroValue) -> Result<usize> {
7731    BASE64
7732        .decode(encode_avro_value(value)?.blob)
7733        .map(|bytes| bytes.len())
7734        .map_err(|error| Error::Codec(format!("memo Avro encoding was not strict base64: {error}")))
7735}
7736
7737fn canonical_memo_entries(value: AvroValue, require_entries: bool) -> Result<AvroValue> {
7738    let AvroValue::Map(entries) = value else {
7739        return Err(Error::InvalidMemoUpdate(
7740            "entries must serialize to an Avro string-keyed map".to_string(),
7741        ));
7742    };
7743    if require_entries && entries.is_empty() {
7744        return Err(Error::InvalidMemoUpdate(
7745            "at least one entry is required".to_string(),
7746        ));
7747    }
7748    if entries.len() > MAX_MEMO_ENTRIES {
7749        return Err(Error::InvalidMemoUpdate(format!(
7750            "at most {MAX_MEMO_ENTRIES} entries are allowed"
7751        )));
7752    }
7753
7754    for (key, value) in &entries {
7755        if !valid_memo_key(&key) {
7756            return Err(Error::InvalidMemoUpdate(
7757                "keys must match ^(?!-?[0-9]+$)[A-Za-z0-9_.:-]{1,64}$".to_string(),
7758            ));
7759        }
7760        if avro_encoded_size(value)? > MAX_MEMO_VALUE_SIZE_BYTES {
7761            return Err(Error::InvalidMemoUpdate(format!(
7762                "value {key:?} exceeds the {MAX_MEMO_VALUE_SIZE_BYTES}-byte limit"
7763            )));
7764        }
7765    }
7766
7767    let value = AvroValue::Map(entries);
7768    if avro_encoded_size(&value)? > MAX_MEMO_TOTAL_SIZE_BYTES {
7769        return Err(Error::InvalidMemoUpdate(format!(
7770            "update exceeds the {MAX_MEMO_TOTAL_SIZE_BYTES}-byte total limit"
7771        )));
7772    }
7773    Ok(value)
7774}
7775
7776fn decode_memo_history_map(envelope: &Value, require_entries: bool) -> Result<AvroValue> {
7777    let object = envelope.as_object().ok_or_else(|| {
7778        Error::InvalidMemoUpdate(
7779            "history field must use the public {codec, blob} payload envelope".to_string(),
7780        )
7781    })?;
7782    if object.len() != 2 || !object.contains_key("codec") || !object.contains_key("blob") {
7783        return Err(Error::InvalidMemoUpdate(
7784            "history field must use exactly the public {codec, blob} payload envelope".to_string(),
7785        ));
7786    }
7787
7788    canonical_memo_entries(
7789        decode_wire_avro_value(envelope, DEFAULT_CODEC)?,
7790        require_entries,
7791    )
7792}
7793
7794impl WorkflowContext {
7795    pub fn message_stream(&self, name: impl Into<String>) -> Result<MessageStream> {
7796        let name = name.into();
7797        if name.is_empty()
7798            || name.len() > 128
7799            || !name.bytes().all(|byte| {
7800                byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b':' | b'-')
7801            })
7802        {
7803            return Err(Error::Codec(
7804                "message stream names must contain 1-128 letters, numbers, periods, underscores, colons, or hyphens"
7805                    .to_string(),
7806            ));
7807        }
7808        Ok(MessageStream {
7809            ctx: self.clone(),
7810            name,
7811        })
7812    }
7813
7814    fn record_message_stream_wait(&self, name: &str) -> Result<()> {
7815        let mut state = self
7816            .state
7817            .lock()
7818            .map_err(|_| Error::WorkflowStatePoisoned)?;
7819        let position = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7820        state
7821            .message_stream_waits
7822            .insert(name.to_string(), position);
7823        Ok(())
7824    }
7825
7826    fn buffer_message_stream(&self, message: MessageStreamMessage) -> Result<()> {
7827        let mut state = self
7828            .state
7829            .lock()
7830            .map_err(|_| Error::WorkflowStatePoisoned)?;
7831        let cursor = state
7832            .message_stream_cursors
7833            .get(&message.stream_name)
7834            .copied()
7835            .unwrap_or(0);
7836        if message.position <= cursor {
7837            return Ok(());
7838        }
7839        let pending = state
7840            .message_stream_messages
7841            .entry(message.stream_name.clone())
7842            .or_default();
7843        if pending.iter().any(|candidate| {
7844            candidate.position == message.position || candidate.message_id == message.message_id
7845        }) {
7846            return Ok(());
7847        }
7848        pending.push(message);
7849        pending.sort_by_key(|candidate| candidate.position);
7850        Ok(())
7851    }
7852
7853    fn buffer_message_stream_delivery(&self, arguments: Vec<Value>) -> Result<Option<String>> {
7854        if let Some(delivery) = decode_message_stream_delivery(arguments)? {
7855            match delivery {
7856                MessageStreamDelivery::Message(message) => {
7857                    let stream_name = message.stream_name.clone();
7858                    self.buffer_message_stream(message)?;
7859                    return Ok(Some(stream_name));
7860                }
7861                MessageStreamDelivery::Cursor {
7862                    stream_name,
7863                    through_position,
7864                } => self.apply_message_stream_cursor(&stream_name, through_position)?,
7865            }
7866        }
7867        Ok(None)
7868    }
7869
7870    fn next_message_stream_wait_sequence(&self) -> Result<Option<u64>> {
7871        let state = self
7872            .state
7873            .lock()
7874            .map_err(|_| Error::WorkflowStatePoisoned)?;
7875        Ok(match state.recorded_commands.get(state.command_cursor) {
7876            Some(RecordedCommand::SignalWait {
7877                sequence,
7878                signal_name,
7879                ..
7880            }) if signal_name == MESSAGE_STREAM_SIGNAL => Some(*sequence),
7881            _ => None,
7882        })
7883    }
7884
7885    fn buffer_message_stream_history_for_wait(&self, wait_sequence: u64) -> Result<()> {
7886        let (history, payload_codec) = {
7887            let state = self
7888                .state
7889                .lock()
7890                .map_err(|_| Error::WorkflowStatePoisoned)?;
7891            (
7892                Arc::clone(&state.history_events),
7893                state.payload_codec.clone(),
7894            )
7895        };
7896
7897        let Some(opened_index) = history.iter().position(|event| {
7898            event.event_type == "SignalWaitOpened"
7899                && durable_event_sequence(event) == Some(wait_sequence)
7900                && event.payload.get("signal_name").and_then(Value::as_str)
7901                    == Some(MESSAGE_STREAM_SIGNAL)
7902        }) else {
7903            return Ok(());
7904        };
7905        let boundary_index = history
7906            .iter()
7907            .enumerate()
7908            .skip(opened_index + 1)
7909            .find_map(|(index, event)| {
7910                (durable_event_sequence(event).is_some_and(|sequence| sequence > wait_sequence)
7911                    && is_authored_command_open_event(event))
7912                .then_some(index)
7913            })
7914            .unwrap_or(history.len());
7915
7916        for event in history[opened_index + 1..boundary_index]
7917            .iter()
7918            .filter(|event| {
7919                event.event_type == "SignalReceived"
7920                    && event.payload.get("signal_name").and_then(Value::as_str)
7921                        == Some(MESSAGE_STREAM_SIGNAL)
7922            })
7923        {
7924            let arguments = decode_signal_event_arguments(event, &payload_codec)?
7925                .into_iter()
7926                .map(AvroValue::into_json)
7927                .collect::<Result<Vec<_>>>()?;
7928            self.buffer_message_stream_delivery(arguments)?;
7929        }
7930        Ok(())
7931    }
7932
7933    fn apply_message_stream_cursor(&self, name: &str, through_position: u64) -> Result<()> {
7934        let mut state = self
7935            .state
7936            .lock()
7937            .map_err(|_| Error::WorkflowStatePoisoned)?;
7938        let cursor = state
7939            .message_stream_cursors
7940            .entry(name.to_string())
7941            .or_default();
7942        *cursor = (*cursor).max(through_position);
7943        if let Some(pending) = state.message_stream_messages.get_mut(name) {
7944            pending.retain(|message| message.position > through_position);
7945        }
7946        Ok(())
7947    }
7948
7949    fn take_message_stream_batch(
7950        &self,
7951        name: &str,
7952        max_items: usize,
7953    ) -> Result<Option<Vec<MessageStreamMessage>>> {
7954        let mut state = self
7955            .state
7956            .lock()
7957            .map_err(|_| Error::WorkflowStatePoisoned)?;
7958        let cursor = state.message_stream_cursors.get(name).copied().unwrap_or(0);
7959        let pending = state
7960            .message_stream_messages
7961            .entry(name.to_string())
7962            .or_default();
7963        let count = contiguous_message_stream_count(pending, cursor, max_items);
7964        if count == 0 {
7965            return Ok(None);
7966        }
7967        let batch = pending.drain(..count).collect::<Vec<_>>();
7968        let position = batch.last().map(|message| message.position).unwrap_or(0);
7969        state
7970            .message_stream_cursors
7971            .insert(name.to_string(), position);
7972        state.message_stream_waits.remove(name);
7973        Ok(Some(batch))
7974    }
7975
7976    fn message_stream_metadata(&self) -> Result<(Vec<Value>, Vec<Value>)> {
7977        let state = self
7978            .state
7979            .lock()
7980            .map_err(|_| Error::WorkflowStatePoisoned)?;
7981        let mut cursors = state.message_stream_cursors.iter().collect::<Vec<_>>();
7982        cursors.sort_by_key(|(name, _)| *name);
7983        let mut waits = state.message_stream_waits.iter().collect::<Vec<_>>();
7984        waits.sort_by_key(|(name, _)| *name);
7985        Ok((
7986            cursors
7987                .into_iter()
7988                .map(|(name, position)| json!({"stream_name": name, "through_position": position}))
7989                .collect(),
7990            waits
7991                .into_iter()
7992                .map(|(name, position)| json!({"stream_name": name, "after_position": position}))
7993                .collect(),
7994        ))
7995    }
7996    /// Identity of the parent workflow currently being replayed.
7997    pub fn workflow_identity(&self) -> Result<WorkflowIdentity> {
7998        let state = self
7999            .state
8000            .lock()
8001            .map_err(|_| Error::WorkflowStatePoisoned)?;
8002        Ok(WorkflowIdentity {
8003            workflow_id: state.workflow_id.clone(),
8004            run_id: state.run_id.clone(),
8005        })
8006    }
8007
8008    /// Return the server-published history budget for this workflow task.
8009    pub fn history_budget(&self) -> Result<WorkflowHistoryBudget> {
8010        let state = self
8011            .state
8012            .lock()
8013            .map_err(|_| Error::WorkflowStatePoisoned)?;
8014        Ok(state.history_budget.clone())
8015    }
8016
8017    /// Continue this workflow instance as a fresh run with replacement arguments.
8018    ///
8019    /// Return this value directly from the workflow handler. The worker converts
8020    /// it to the terminal protocol command only after replay has consumed every
8021    /// recorded durable command.
8022    pub fn continue_as_new<T: Serialize>(&self, args: T) -> Result<Value> {
8023        self.continue_as_new_with_options(ContinueAsNewOptions::new(), args)
8024    }
8025
8026    /// Continue as new with optional workflow-type and task-queue overrides.
8027    pub fn continue_as_new_with_options<T: Serialize>(
8028        &self,
8029        options: ContinueAsNewOptions,
8030        args: T,
8031    ) -> Result<Value> {
8032        options.validate()?;
8033        Err(Error::ContinueAsNew(ContinueAsNewRequest {
8034            arguments: normalize_avro_arguments(AvroValue::from_serialize(&args)?),
8035            options,
8036        }))
8037    }
8038
8039    pub fn activity<T: Serialize>(
8040        &self,
8041        activity_type: impl Into<String>,
8042        args: T,
8043    ) -> ActivityCall {
8044        self.activity_with_options(activity_type, ActivityOptions::new(), args)
8045    }
8046
8047    pub fn activity_on_queue<T, Q>(
8048        &self,
8049        activity_type: impl Into<String>,
8050        task_queue: Option<Q>,
8051        args: T,
8052    ) -> ActivityCall
8053    where
8054        T: Serialize,
8055        Q: Into<String>,
8056    {
8057        let mut options = ActivityOptions::new();
8058        options.task_queue = task_queue.map(Into::into);
8059        self.activity_with_options(activity_type, options, args)
8060    }
8061
8062    /// Schedule one durable activity with retry, routing, and timeout options.
8063    ///
8064    /// Options are validated before the command is emitted. Once the command is
8065    /// recorded, replay consumes the same activity lifecycle at this command
8066    /// position and never emits a duplicate schedule.
8067    ///
8068    /// ```no_run
8069    /// # use durable_workflow::{json, ActivityOptions, ActivityRetryPolicy, Error, Result, WorkflowContext};
8070    /// # use std::time::Duration;
8071    /// # async fn run(ctx: WorkflowContext) -> Result<durable_workflow::Value> {
8072    /// let result = ctx
8073    ///     .activity_with_options(
8074    ///         "charge-card",
8075    ///         ActivityOptions::new()
8076    ///             .task_queue("payments")
8077    ///             .retry_policy(
8078    ///                 ActivityRetryPolicy::new(4).exponential_backoff(
8079    ///                     Duration::from_secs(1),
8080    ///                     2,
8081    ///                     Some(Duration::from_secs(30)),
8082    ///                 ),
8083    ///             )
8084    ///             .start_to_close_timeout(Duration::from_secs(60))
8085    ///             .schedule_to_close_timeout(Duration::from_secs(180))
8086    ///             .heartbeat_timeout(Duration::from_secs(15)),
8087    ///         json!([{"order_id": "order-42"}]),
8088    ///     )
8089    ///     .await;
8090    /// match result {
8091    ///     Err(Error::ActivityFailed(failure)) => Ok(json!({
8092    ///         "reason": failure.reason,
8093    ///         "timeout_kind": failure.timeout_kind,
8094    ///     })),
8095    ///     other => other,
8096    /// }
8097    /// # }
8098    /// ```
8099    pub fn activity_with_options<T: Serialize>(
8100        &self,
8101        activity_type: impl Into<String>,
8102        options: ActivityOptions,
8103        args: T,
8104    ) -> ActivityCall {
8105        ActivityCall {
8106            ctx: self.clone(),
8107            activity_type: activity_type.into(),
8108            options,
8109            args: Some(AvroValue::from_serialize(&args)),
8110            scheduled: false,
8111            parallel_group_path: Vec::new(),
8112        }
8113    }
8114
8115    pub async fn activity_avro_value<T: Serialize>(
8116        &self,
8117        activity_type: impl Into<String>,
8118        args: T,
8119    ) -> Result<AvroValue> {
8120        let mut call = self.activity(activity_type, args);
8121        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8122    }
8123
8124    pub async fn activity_avro_value_with_options<T: Serialize>(
8125        &self,
8126        activity_type: impl Into<String>,
8127        options: ActivityOptions,
8128        args: T,
8129    ) -> Result<AvroValue> {
8130        let mut call = self.activity_with_options(activity_type, options, args);
8131        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8132    }
8133
8134    /// Schedule an activity with a Serde request and decode its Serde result.
8135    pub async fn activity_typed<I, O>(&self, activity_type: impl Into<String>, args: I) -> Result<O>
8136    where
8137        I: Serialize,
8138        O: DeserializeOwned,
8139    {
8140        self.activity_typed_with_options(activity_type, ActivityOptions::new(), args)
8141            .await
8142    }
8143
8144    /// Schedule an activity with options and decode its result into `O`.
8145    ///
8146    /// Both directions use the fixed Avro Value codec. In particular, this
8147    /// method does not deserialize the JSON-safe inspection projection returned
8148    /// by the dynamic [`ActivityCall`] future.
8149    pub async fn activity_typed_with_options<I, O>(
8150        &self,
8151        activity_type: impl Into<String>,
8152        options: ActivityOptions,
8153        args: I,
8154    ) -> Result<O>
8155    where
8156        I: Serialize,
8157        O: DeserializeOwned,
8158    {
8159        let activity_type = activity_type.into();
8160        let encoded = AvroValue::from_serialize(&args).map_err(|error| {
8161            handler_type_error::<I>(
8162                HandlerKind::Activity,
8163                &activity_type,
8164                HandlerValueKind::Input,
8165                error.to_string(),
8166            )
8167        });
8168        let mut call = ActivityCall {
8169            ctx: self.clone(),
8170            activity_type: activity_type.clone(),
8171            options,
8172            args: Some(encoded),
8173            scheduled: false,
8174            parallel_group_path: Vec::new(),
8175        };
8176        let result = std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await?;
8177        decode_handler_result(result, HandlerKind::Activity, &activity_type)
8178    }
8179
8180    /// Schedule and join a deterministic activity/child/timer group.
8181    ///
8182    /// Nested groups retain their input shape. Every durable leaf is scheduled
8183    /// before this future yields, results are assembled by declaration order,
8184    /// and a failure returns [`Error::ParallelFailed`] with typed cause,
8185    /// declaration path, stable group metadata, and completed siblings.
8186    pub fn parallel(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8187        ParallelCall::new(self.clone(), operations)
8188    }
8189
8190    /// Alias for [`WorkflowContext::parallel`].
8191    pub fn join(&self, operations: Vec<ParallelOperation>) -> ParallelCall {
8192        self.parallel(operations)
8193    }
8194
8195    /// Lossless fixed-Avro variant of [`WorkflowContext::parallel`].
8196    pub async fn parallel_avro_value(
8197        &self,
8198        operations: Vec<ParallelOperation>,
8199    ) -> Result<Vec<ParallelAvroResult>> {
8200        let mut call = self.parallel(operations);
8201        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8202    }
8203
8204    /// Start every durable operation and resume from the one winner persisted
8205    /// by Server. Non-winning operations continue and remain addressable.
8206    pub fn select(&self, operations: Vec<ParallelOperation>) -> SelectCall {
8207        let operations = operations
8208            .into_iter()
8209            .enumerate()
8210            .map(|(index, operation)| (SelectionKey::Index(index), operation))
8211            .collect();
8212        SelectCall::new(self.clone(), operations)
8213    }
8214
8215    /// Named-key variant of [`WorkflowContext::select`].
8216    pub fn select_keyed<K>(&self, operations: Vec<(K, ParallelOperation)>) -> SelectCall
8217    where
8218        K: Into<SelectionKey>,
8219    {
8220        SelectCall::new(
8221            self.clone(),
8222            operations
8223                .into_iter()
8224                .map(|(key, operation)| (key.into(), operation))
8225                .collect(),
8226        )
8227    }
8228
8229    /// Create a workflow-local deterministic compensation registry.
8230    pub fn saga(&self) -> Saga {
8231        Saga::new(self.clone())
8232    }
8233
8234    /// Whether the current workflow task carries a cooperative cancel request.
8235    pub fn is_cancellation_requested(&self) -> Result<bool> {
8236        let state = self
8237            .state
8238            .lock()
8239            .map_err(|_| Error::WorkflowStatePoisoned)?;
8240        Ok(state.cancel_requested)
8241    }
8242
8243    /// Raise a typed cooperative cancellation at an author-controlled point.
8244    ///
8245    /// Passing this result to [`Saga::finish`] compensates already registered
8246    /// forward steps before the cancellation remains the initiating outcome.
8247    pub fn throw_if_cancellation_requested(&self) -> Result<()> {
8248        if self.is_cancellation_requested()? {
8249            return Err(Error::WorkflowCancellationRequested(
8250                WorkflowCancellationRequested,
8251            ));
8252        }
8253        Ok(())
8254    }
8255
8256    pub fn wait_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8257        SignalCall {
8258            ctx: self.clone(),
8259            signal_name: signal_name.into(),
8260            runtime_reserved_allowed: false,
8261            opened_wait: false,
8262            matched_pending: false,
8263            parallel_group_path: Vec::new(),
8264        }
8265    }
8266
8267    fn wait_runtime_signal(&self, signal_name: impl Into<String>) -> SignalCall {
8268        SignalCall {
8269            ctx: self.clone(),
8270            signal_name: signal_name.into(),
8271            runtime_reserved_allowed: true,
8272            opened_wait: false,
8273            matched_pending: false,
8274            parallel_group_path: Vec::new(),
8275        }
8276    }
8277
8278    pub async fn wait_signal_avro_value(
8279        &self,
8280        signal_name: impl Into<String>,
8281    ) -> Result<Vec<AvroValue>> {
8282        let mut call = self.wait_signal(signal_name);
8283        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8284    }
8285
8286    /// Return every committed signal argument list with the given name.
8287    ///
8288    /// This history-backed view is deterministic and is intended for
8289    /// condition predicates that must be re-evaluated after a signal while the
8290    /// workflow is blocked on [`WorkflowContext::wait_condition`].
8291    pub fn signals(&self, signal_name: &str) -> Result<Vec<Vec<Value>>> {
8292        self.signals_avro_value(signal_name)?
8293            .into_iter()
8294            .map(|arguments| {
8295                arguments
8296                    .into_iter()
8297                    .map(AvroValue::into_json)
8298                    .collect::<Result<Vec<_>>>()
8299            })
8300            .collect()
8301    }
8302
8303    /// Lossless fixed Avro Value view of committed signals with the given name.
8304    pub fn signals_avro_value(&self, signal_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8305        let state = self
8306            .state
8307            .lock()
8308            .map_err(|_| Error::WorkflowStatePoisoned)?;
8309        state
8310            .history_events
8311            .iter()
8312            .filter(|event| {
8313                event.event_type == "SignalReceived"
8314                    && event.payload.get("signal_name").and_then(Value::as_str) == Some(signal_name)
8315            })
8316            .map(|event| decode_signal_event_arguments(event, &state.payload_codec))
8317            .collect()
8318    }
8319
8320    /// Return every committed update argument list with the given name.
8321    ///
8322    /// Accepted and applied records for the same update ID are de-duplicated.
8323    /// A Server task created after an update therefore replays the workflow and
8324    /// re-evaluates an open condition without application polling.
8325    pub fn updates(&self, update_name: &str) -> Result<Vec<Vec<Value>>> {
8326        self.updates_avro_value(update_name)?
8327            .into_iter()
8328            .map(|arguments| {
8329                arguments
8330                    .into_iter()
8331                    .map(AvroValue::into_json)
8332                    .collect::<Result<Vec<_>>>()
8333            })
8334            .collect()
8335    }
8336
8337    /// Lossless fixed Avro Value view of committed updates with the given name.
8338    pub fn updates_avro_value(&self, update_name: &str) -> Result<Vec<Vec<AvroValue>>> {
8339        let state = self
8340            .state
8341            .lock()
8342            .map_err(|_| Error::WorkflowStatePoisoned)?;
8343        let mut seen = Vec::new();
8344        let mut updates = Vec::new();
8345        for event in state.history_events.iter() {
8346            if !matches!(
8347                event.event_type.as_str(),
8348                "UpdateAccepted" | "UpdateApplied"
8349            ) || event.payload.get("update_name").and_then(Value::as_str) != Some(update_name)
8350                || event.payload.get("arguments").is_none()
8351            {
8352                continue;
8353            }
8354            if let Some(update_id) = event.payload.get("update_id").and_then(Value::as_str) {
8355                if seen.iter().any(|recorded| recorded == update_id) {
8356                    continue;
8357                }
8358                seen.push(update_id.to_string());
8359            }
8360            updates.push(decode_update_event_arguments(event, &state.payload_codec)?);
8361        }
8362        Ok(updates)
8363    }
8364
8365    /// Wait for a deterministic predicate to become true or for its durable
8366    /// timeout to elapse.
8367    ///
8368    /// Prefer [`wait_condition!`] for inline predicates so changes to the Rust
8369    /// predicate tokens automatically change the recorded definition
8370    /// fingerprint. Direct callers must provide an equally stable identity in
8371    /// [`ConditionWaitOptions`].
8372    pub fn wait_condition<F>(
8373        &self,
8374        options: ConditionWaitOptions,
8375        predicate: F,
8376    ) -> ConditionWaitCall
8377    where
8378        F: Fn() -> Result<bool> + Send + 'static,
8379    {
8380        ConditionWaitCall {
8381            ctx: self.clone(),
8382            options,
8383            predicate: Box::new(predicate),
8384            occurrence_id: None,
8385            opened_wait: false,
8386            parallel_group_path: Vec::new(),
8387        }
8388    }
8389
8390    /// Wait for server-backed durable time without blocking the worker executor.
8391    ///
8392    /// Polling this future emits one `start_timer` command and yields. The
8393    /// server records the deadline, so neither worker nor server restarts reset
8394    /// the wait. Replay resolves the future only from a `TimerScheduled` and
8395    /// `TimerFired` pair at the same position in the shared durable-command
8396    /// stream, with matching sequence, timer identity, and delay. Sub-second
8397    /// durations round up because protocol deadlines use whole seconds.
8398    ///
8399    /// ```no_run
8400    /// # use durable_workflow::{json, Client, Worker};
8401    /// # use std::time::Duration;
8402    /// # fn configure(client: Client) {
8403    /// let mut worker = Worker::new(client, "rust-workers");
8404    /// worker.register_workflow("delayed-greeting", |ctx, _input| async move {
8405    ///     ctx.sleep(Duration::from_secs(5)).await?;
8406    ///     Ok(json!({"status": "timer fired"}))
8407    /// });
8408    /// # }
8409    /// ```
8410    pub fn sleep(&self, duration: Duration) -> TimerCall {
8411        let delay_seconds = duration
8412            .as_secs()
8413            .checked_add(u64::from(duration.subsec_nanos() > 0));
8414        TimerCall {
8415            ctx: self.clone(),
8416            delay_seconds,
8417            scheduled: false,
8418            matched_pending: false,
8419            parallel_group_path: Vec::new(),
8420        }
8421    }
8422
8423    /// Alias for [`WorkflowContext::sleep`] for timer-oriented workflow code.
8424    pub fn start_timer(&self, duration: Duration) -> TimerCall {
8425        self.sleep(duration)
8426    }
8427
8428    /// Evaluate a non-deterministic callback once and durably record its typed value.
8429    ///
8430    /// On replay the callback is not invoked: the value is decoded from the
8431    /// sequence-matched `SideEffectRecorded` event using the workflow's payload
8432    /// codec. Use this for UUIDs, wall-clock snapshots, random values, and other
8433    /// small values that must remain fixed for the lifetime of a workflow run.
8434    pub fn side_effect<T, F>(&self, callback: F) -> Result<T>
8435    where
8436        T: Serialize + DeserializeOwned,
8437        F: FnOnce() -> T,
8438    {
8439        {
8440            let mut state = self
8441                .state
8442                .lock()
8443                .map_err(|_| Error::WorkflowStatePoisoned)?;
8444            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8445                return match recorded {
8446                    RecordedCommand::SideEffect { sequence, value } => {
8447                        state.command_cursor += 1;
8448                        value.deserialize().map_err(|error| {
8449                            Error::NonDeterministicReplay(ReplayFailure::new(
8450                                "side_effect_type_mismatch",
8451                                Some(sequence),
8452                                Some(std::any::type_name::<T>().to_string()),
8453                                Some(error.to_string()),
8454                                "recorded side-effect value is incompatible with the requested Rust type",
8455                            ))
8456                        })
8457                    }
8458                    other => Err(command_mismatch(&other, "side effect")),
8459                };
8460            }
8461        }
8462
8463        let value = callback();
8464        let avro_value = AvroValue::from_serialize(&value)?;
8465        let mut state = self
8466            .state
8467            .lock()
8468            .map_err(|_| Error::WorkflowStatePoisoned)?;
8469        let result = encode_typed_envelope(&avro_value, &state.payload_codec)?;
8470        state.commands.push(json!({
8471            "type": "record_side_effect",
8472            "result": result,
8473        }));
8474        Ok(value)
8475    }
8476
8477    /// Record or replay a lossless fixed Avro Value side effect.
8478    pub fn side_effect_avro_value<F>(&self, callback: F) -> Result<AvroValue>
8479    where
8480        F: FnOnce() -> AvroValue,
8481    {
8482        {
8483            let mut state = self
8484                .state
8485                .lock()
8486                .map_err(|_| Error::WorkflowStatePoisoned)?;
8487            if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8488                return match recorded {
8489                    RecordedCommand::SideEffect { value, .. } => {
8490                        state.command_cursor += 1;
8491                        Ok(value)
8492                    }
8493                    other => Err(command_mismatch(&other, "side effect")),
8494                };
8495            }
8496        }
8497
8498        let value = callback();
8499        let mut state = self
8500            .state
8501            .lock()
8502            .map_err(|_| Error::WorkflowStatePoisoned)?;
8503        let result = encode_typed_envelope(&value, &state.payload_codec)?;
8504        state.commands.push(json!({
8505            "type": "record_side_effect",
8506            "result": result,
8507        }));
8508        Ok(value)
8509    }
8510
8511    /// Append output items at a deterministic workflow command boundary.
8512    ///
8513    /// Stable idempotency keys are derived from the server-provided durable
8514    /// workflow command identity, command ordinal, and item index. Replay
8515    /// consumes the recorded side effect and never emits another append.
8516    pub fn append_workflow_stream(
8517        &self,
8518        stream_name: impl Into<String>,
8519        items: &[WorkflowStreamAppendItem],
8520        max_pending_items: Option<u64>,
8521    ) -> Result<()> {
8522        if items.is_empty() {
8523            return Err(Error::Codec(
8524                "workflow_stream_items_empty: append requires at least one item".to_string(),
8525            ));
8526        }
8527        if max_pending_items == Some(0) {
8528            return Err(Error::Codec(
8529                "workflow_stream_pending_limit_invalid: max_pending_items must be positive"
8530                    .to_string(),
8531            ));
8532        }
8533        let stream_name = stream_name.into();
8534        if stream_name.is_empty() {
8535            return Err(Error::Codec(
8536                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8537            ));
8538        }
8539
8540        let mut state = self
8541            .state
8542            .lock()
8543            .map_err(|_| Error::WorkflowStatePoisoned)?;
8544        let command_ordinal = state.workflow_stream_command_counter;
8545        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8546            state.workflow_stream_command_counter += 1;
8547            return match recorded {
8548                RecordedCommand::SideEffect { .. } => {
8549                    state.command_cursor += 1;
8550                    Ok(())
8551                }
8552                other => Err(command_mismatch(&other, "workflow stream append")),
8553            };
8554        }
8555
8556        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8557        state.workflow_stream_command_counter += 1;
8558        let wire_items = items
8559            .iter()
8560            .enumerate()
8561            .map(|(item_index, item)| {
8562                item.wire_value(Some(format!(
8563                    "dw-stream:{identity}:{command_ordinal}:{item_index}"
8564                )))
8565            })
8566            .collect::<Vec<_>>();
8567        let mut directive = json!({
8568            "operation": "append",
8569            "stream_name": stream_name,
8570            "command_identity": identity,
8571            "command_ordinal": command_ordinal,
8572            "items": wire_items,
8573        });
8574        if let Some(max_pending_items) = max_pending_items {
8575            directive["max_pending_items"] = json!(max_pending_items);
8576        }
8577        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8578        state.commands.push(json!({
8579            "type": "record_side_effect",
8580            "result": result,
8581            "workflow_stream": directive,
8582        }));
8583        Ok(())
8584    }
8585
8586    /// Close a run-scoped output stream at a deterministic command boundary.
8587    pub fn close_workflow_stream(
8588        &self,
8589        stream_name: impl Into<String>,
8590        retention_seconds: Option<u64>,
8591    ) -> Result<()> {
8592        self.finish_workflow_stream(stream_name.into(), None, retention_seconds)
8593    }
8594
8595    /// Mark a run-scoped output stream errored at a deterministic command boundary.
8596    pub fn error_workflow_stream(
8597        &self,
8598        stream_name: impl Into<String>,
8599        error_reason: impl Into<String>,
8600        retention_seconds: Option<u64>,
8601    ) -> Result<()> {
8602        let error_reason = error_reason.into();
8603        if error_reason.is_empty() {
8604            return Err(Error::Codec(
8605                "workflow_stream_error_invalid: error reason must not be empty".to_string(),
8606            ));
8607        }
8608        self.finish_workflow_stream(stream_name.into(), Some(error_reason), retention_seconds)
8609    }
8610
8611    fn finish_workflow_stream(
8612        &self,
8613        stream_name: String,
8614        error_reason: Option<String>,
8615        retention_seconds: Option<u64>,
8616    ) -> Result<()> {
8617        if stream_name.is_empty() {
8618            return Err(Error::Codec(
8619                "workflow_stream_name_invalid: stream name must not be empty".to_string(),
8620            ));
8621        }
8622        if retention_seconds == Some(0) {
8623            return Err(Error::Codec(
8624                "workflow_stream_retention_invalid: retention_seconds must be positive".to_string(),
8625            ));
8626        }
8627        let mut state = self
8628            .state
8629            .lock()
8630            .map_err(|_| Error::WorkflowStatePoisoned)?;
8631        let command_ordinal = state.workflow_stream_command_counter;
8632        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8633            state.workflow_stream_command_counter += 1;
8634            return match recorded {
8635                RecordedCommand::SideEffect { .. } => {
8636                    state.command_cursor += 1;
8637                    Ok(())
8638                }
8639                other => Err(command_mismatch(&other, "workflow stream close")),
8640            };
8641        }
8642        let identity = Self::workflow_stream_command_identity(&state)?.to_string();
8643        state.workflow_stream_command_counter += 1;
8644        let mut directive = json!({
8645            "operation": if error_reason.is_some() { "error" } else { "close" },
8646            "stream_name": stream_name,
8647            "command_identity": identity,
8648            "command_ordinal": command_ordinal,
8649        });
8650        if let Some(error_reason) = error_reason {
8651            directive["error_reason"] = json!(error_reason);
8652        }
8653        if let Some(retention_seconds) = retention_seconds {
8654            directive["retention_seconds"] = json!(retention_seconds);
8655        }
8656        let result = encode_typed_envelope(&AvroValue::Null, &state.payload_codec)?;
8657        state.commands.push(json!({
8658            "type": "record_side_effect",
8659            "result": result,
8660            "workflow_stream": directive,
8661        }));
8662        Ok(())
8663    }
8664
8665    fn workflow_stream_command_identity(state: &WorkflowState) -> Result<&str> {
8666        let identity = state.workflow_command_identity.as_str();
8667        if identity.is_empty() {
8668            return Err(Error::MissingWorkflowCommandIdentity);
8669        }
8670        Ok(identity)
8671    }
8672
8673    /// Validate, emit, or replay a typed workflow search-attribute update.
8674    ///
8675    /// The command is non-blocking within a workflow decision, but its
8676    /// `SearchAttributesUpserted` event occupies the same deterministic command
8677    /// stream as activities, timers, conditions, and other durable operations.
8678    pub fn upsert_search_attributes(&self, update: SearchAttributeUpdate) -> Result<()> {
8679        update.validate()?;
8680        let (attributes, attribute_types) = update.into_wire_parts();
8681        let mut state = self
8682            .state
8683            .lock()
8684            .map_err(|_| Error::WorkflowStatePoisoned)?;
8685
8686        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8687            return match recorded {
8688                RecordedCommand::SearchAttributes {
8689                    sequence,
8690                    attributes: recorded_attributes,
8691                    attribute_types: recorded_attribute_types,
8692                } => {
8693                    if recorded_attributes != attributes {
8694                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8695                            "search_attribute_value_mismatch",
8696                            Some(sequence),
8697                            Some(recorded_attributes.to_string()),
8698                            Some(attributes.to_string()),
8699                            "search-attribute values differ from the recorded durable command",
8700                        )));
8701                    }
8702                    if let RecordedSnapshotValue::Known(recorded_types) = recorded_attribute_types {
8703                        if recorded_types != attribute_types {
8704                            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8705                                "search_attribute_type_mismatch",
8706                                Some(sequence),
8707                                Some(json!(recorded_types).to_string()),
8708                                Some(json!(attribute_types).to_string()),
8709                                "search-attribute declared types differ from the recorded durable command",
8710                            )));
8711                        }
8712                    }
8713                    state.command_cursor += 1;
8714                    Ok(())
8715                }
8716                other => Err(command_mismatch(&other, "search-attribute update")),
8717            };
8718        }
8719
8720        let mut command = serde_json::Map::from_iter([
8721            ("type".to_string(), json!("upsert_search_attributes")),
8722            ("attributes".to_string(), attributes),
8723        ]);
8724        if !attribute_types.is_empty() {
8725            command.insert("attribute_types".to_string(), json!(attribute_types));
8726        }
8727        state.commands.push(Value::Object(command));
8728        Ok(())
8729    }
8730
8731    /// Record a UUIDv4 once and return the same UUID on every replay.
8732    pub fn uuid_v4(&self) -> Result<Uuid> {
8733        self.side_effect(Uuid::new_v4)
8734    }
8735
8736    /// Select the newest supported version for a change, or replay the version
8737    /// already committed for that stable change ID.
8738    pub fn get_version(
8739        &self,
8740        change_id: impl Into<String>,
8741        min_supported: i32,
8742        max_supported: i32,
8743    ) -> Result<i32> {
8744        let change_id = change_id.into();
8745        if change_id.trim().is_empty() {
8746            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8747                "version_change_id_invalid",
8748                None,
8749                Some("non-empty change ID".to_string()),
8750                Some(change_id),
8751                "version markers require a stable non-empty change ID",
8752            )));
8753        }
8754        if min_supported > max_supported {
8755            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8756                "version_range_invalid",
8757                None,
8758                Some("min_supported <= max_supported".to_string()),
8759                Some(format!("{min_supported}..={max_supported}")),
8760                "version marker supported range is invalid",
8761            )));
8762        }
8763
8764        let mut state = self
8765            .state
8766            .lock()
8767            .map_err(|_| Error::WorkflowStatePoisoned)?;
8768        if let Some((version, sequence)) = state.version_markers.get(&change_id).copied() {
8769            ensure_version_supported(&change_id, version, min_supported, max_supported, sequence)?;
8770            return Ok(version);
8771        }
8772
8773        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8774            return match recorded {
8775                RecordedCommand::VersionMarker {
8776                    sequence,
8777                    change_id: recorded_change_id,
8778                    version,
8779                    ..
8780                } => {
8781                    if recorded_change_id != change_id {
8782                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8783                            "version_change_id_mismatch",
8784                            Some(sequence),
8785                            Some(recorded_change_id),
8786                            Some(change_id),
8787                            "recorded version marker change ID differs from current workflow code",
8788                        )));
8789                    }
8790                    ensure_version_supported(
8791                        &change_id,
8792                        version,
8793                        min_supported,
8794                        max_supported,
8795                        sequence,
8796                    )?;
8797                    state.command_cursor += 1;
8798                    state.version_markers.insert(change_id, (version, sequence));
8799                    Ok(version)
8800                }
8801                other => Err(command_mismatch(
8802                    &other,
8803                    format!("version marker:{change_id}"),
8804                )),
8805            };
8806        }
8807
8808        let version = max_supported;
8809        state.commands.push(json!({
8810            "type": "record_version_marker",
8811            "change_id": change_id,
8812            "version": version,
8813            "min_supported": min_supported,
8814            "max_supported": max_supported,
8815        }));
8816        // Sequence numbers are assigned by the server. Zero identifies a marker
8817        // selected in this uncommitted decision batch for duplicate-call checks.
8818        state.version_markers.insert(change_id, (version, 0));
8819        Ok(version)
8820    }
8821
8822    /// Record or replay the standard `-1` (legacy) / `1` (patched) marker.
8823    pub fn patched(&self, change_id: impl Into<String>) -> Result<bool> {
8824        Ok(self.get_version(change_id, -1, 1)? == 1)
8825    }
8826
8827    /// Keep a patch marker in history after the legacy branch has been removed.
8828    pub fn deprecate_patch(&self, change_id: impl Into<String>) -> Result<()> {
8829        self.get_version(change_id, -1, 1).map(|_| ())
8830    }
8831
8832    /// Merge non-indexed workflow memo metadata through durable history.
8833    ///
8834    /// Avro `null` deletes a key. The SDK encodes the complete patch in the
8835    /// public Avro payload envelope consumed by Server and Cloud runtimes.
8836    pub fn upsert_memo<T: Serialize>(&self, entries: T) -> Result<()> {
8837        let entries = canonical_memo_entries(AvroValue::from_serialize(&entries)?, true)?;
8838        let mut state = self
8839            .state
8840            .lock()
8841            .map_err(|_| Error::WorkflowStatePoisoned)?;
8842
8843        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8844            return match recorded {
8845                RecordedCommand::Memo {
8846                    sequence,
8847                    entries: recorded_entries,
8848                } => {
8849                    if recorded_entries != entries {
8850                        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8851                            "memo_update_mismatch",
8852                            Some(sequence),
8853                            Some(format!("{recorded_entries:?}")),
8854                            Some(format!("{entries:?}")),
8855                            "recorded memo entries differ from the current workflow update",
8856                        )));
8857                    }
8858                    state.command_cursor += 1;
8859                    Ok(())
8860                }
8861                other => Err(command_mismatch(&other, "memo upsert")),
8862            };
8863        }
8864
8865        let entries_envelope = encode_typed_envelope(&entries, DEFAULT_CODEC)?;
8866        state.commands.push(json!({
8867            "type": "upsert_memo",
8868            "entries": entries_envelope,
8869        }));
8870        Ok(())
8871    }
8872
8873    /// Start a named durable child on an explicit queue and await its result.
8874    ///
8875    /// The command is recorded in the parent's sequence-ordered durable command
8876    /// stream. Replay keeps a scheduled child pending without emitting another
8877    /// start, or consumes its matching terminal `ChildRun*` outcome. Successful
8878    /// values preserve the history payload codec and include both sides of the
8879    /// durable relationship; failures are returned as
8880    /// [`Error::ChildWorkflowFailed`].
8881    ///
8882    /// ```no_run
8883    /// # use durable_workflow::{json, ChildWorkflowOptions, Client, ParentClosePolicy, Worker};
8884    /// # fn configure(client: Client) {
8885    /// let mut worker = Worker::new(client, "parent-workers");
8886    /// worker.register_workflow("order-parent", |ctx, _input| async move {
8887    ///     let child = ctx
8888    ///         .start_child_workflow(
8889    ///             "fulfil-order",
8890    ///             ChildWorkflowOptions::new("fulfilment-workers")
8891    ///                 .parent_close_policy(ParentClosePolicy::RequestCancel),
8892    ///             json!([{"order_id": "order-42"}]),
8893    ///         )
8894    ///         .await?;
8895    ///     Ok(child.result)
8896    /// });
8897    /// # }
8898    /// ```
8899    pub fn start_child_workflow<T: Serialize>(
8900        &self,
8901        workflow_type: impl Into<String>,
8902        options: ChildWorkflowOptions,
8903        args: T,
8904    ) -> ChildWorkflowCall {
8905        ChildWorkflowCall {
8906            ctx: self.clone(),
8907            workflow_type: workflow_type.into(),
8908            options,
8909            args: Some(AvroValue::from_serialize(&args)),
8910            scheduled: false,
8911            matched_pending: false,
8912            parallel_group_path: Vec::new(),
8913        }
8914    }
8915
8916    pub async fn start_child_workflow_avro_value<T: Serialize>(
8917        &self,
8918        workflow_type: impl Into<String>,
8919        options: ChildWorkflowOptions,
8920        args: T,
8921    ) -> Result<ChildWorkflowAvroResult> {
8922        let mut call = self.start_child_workflow(workflow_type, options, args);
8923        std::future::poll_fn(|cx| Pin::new(&mut call).poll_avro_value(cx)).await
8924    }
8925
8926    fn take_commands(&self) -> Result<Vec<Value>> {
8927        let mut state = self
8928            .state
8929            .lock()
8930            .map_err(|_| Error::WorkflowStatePoisoned)?;
8931        Ok(std::mem::take(&mut state.commands))
8932    }
8933
8934    fn continue_as_new_command(&self, request: ContinueAsNewRequest) -> Result<Option<Value>> {
8935        let mut state = self
8936            .state
8937            .lock()
8938            .map_err(|_| Error::WorkflowStatePoisoned)?;
8939
8940        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
8941            return Err(command_mismatch(&recorded, "continue as new"));
8942        }
8943        if state.recorded_continue_as_new_sequence.is_some() {
8944            state.continue_as_new_consumed = true;
8945            return Ok(None);
8946        }
8947
8948        let arguments = encode_typed_envelope(&request.arguments, &state.payload_codec)?;
8949        let mut command = serde_json::Map::from_iter([
8950            ("type".to_string(), json!("continue_as_new")),
8951            ("arguments".to_string(), arguments),
8952            ("queue".to_string(), json!(state.task_queue.clone())),
8953        ]);
8954        if let Some(workflow_type) = request.options.workflow_type {
8955            command.insert("workflow_type".to_string(), json!(workflow_type));
8956        }
8957        if let Some(task_queue) = request.options.task_queue {
8958            command.insert("queue".to_string(), json!(task_queue));
8959        }
8960        Ok(Some(Value::Object(command)))
8961    }
8962
8963    fn matched_recorded_pending(&self) -> Result<bool> {
8964        let state = self
8965            .state
8966            .lock()
8967            .map_err(|_| Error::WorkflowStatePoisoned)?;
8968        Ok(state.matched_recorded_pending)
8969    }
8970
8971    fn ensure_history_consumed(&self) -> Result<()> {
8972        let state = self
8973            .state
8974            .lock()
8975            .map_err(|_| Error::WorkflowStatePoisoned)?;
8976        if let Some(command) = state.recorded_commands.get(state.command_cursor) {
8977            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8978                "recorded_commands_unconsumed",
8979                Some(command.sequence()),
8980                Some(command.shape().to_string()),
8981                Some("workflow completion".to_string()),
8982                "workflow completed before consuming all recorded durable commands",
8983            )));
8984        }
8985        if let Some(sequence) = state
8986            .recorded_continue_as_new_sequence
8987            .filter(|_| !state.continue_as_new_consumed)
8988        {
8989            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
8990                "recorded_continue_as_new_unconsumed",
8991                Some(sequence),
8992                Some("continue as new".to_string()),
8993                Some("workflow completion".to_string()),
8994                "workflow completed without consuming its recorded continue-as-new transition",
8995            )));
8996        }
8997        Ok(())
8998    }
8999}
9000
9001fn contiguous_message_stream_count(
9002    pending: &[MessageStreamMessage],
9003    cursor: u64,
9004    max_items: usize,
9005) -> usize {
9006    pending
9007        .iter()
9008        .take(max_items)
9009        .enumerate()
9010        .take_while(|(offset, message)| {
9011            u64::try_from(*offset)
9012                .ok()
9013                .and_then(|offset| cursor.checked_add(offset + 1))
9014                == Some(message.position)
9015        })
9016        .count()
9017}
9018
9019fn is_authored_command_open_event(event: &HistoryEvent) -> bool {
9020    matches!(
9021        event.event_type.as_str(),
9022        "ActivityScheduled"
9023            | "TimerScheduled"
9024            | "ChildWorkflowScheduled"
9025            | "SignalWaitOpened"
9026            | "ConditionWaitOpened"
9027            | "SearchAttributesUpserted"
9028            | "SideEffectRecorded"
9029            | "VersionMarkerRecorded"
9030            | "MemoUpserted"
9031            | "WorkflowContinuedAsNew"
9032    )
9033}
9034
9035#[derive(Debug)]
9036struct WorkflowState {
9037    workflow_id: Option<String>,
9038    run_id: Option<String>,
9039    task_queue: String,
9040    payload_codec: String,
9041    history_events: Arc<Vec<HistoryEvent>>,
9042    history_budget: WorkflowHistoryBudget,
9043    cancel_requested: bool,
9044    resume_signal: Option<ResumeSignal>,
9045    recorded_commands: Vec<RecordedCommand>,
9046    selection_markers: Vec<SelectionMarker>,
9047    selection_marker_cursor: usize,
9048    cancelled_selection_members: Vec<SelectionCancellation>,
9049    recorded_continue_as_new_sequence: Option<u64>,
9050    continue_as_new_consumed: bool,
9051    command_cursor: usize,
9052    condition_wait_occurrence_counter: u64,
9053    matched_recorded_pending: bool,
9054    version_markers: HashMap<String, (i32, u64)>,
9055    workflow_command_identity: String,
9056    workflow_stream_command_counter: u64,
9057    commands: Vec<Value>,
9058    message_stream_messages: HashMap<String, Vec<MessageStreamMessage>>,
9059    message_stream_cursors: HashMap<String, u64>,
9060    message_stream_waits: HashMap<String, u64>,
9061}
9062
9063impl WorkflowState {
9064    #[cfg(test)]
9065    fn new(
9066        history: Vec<HistoryEvent>,
9067        task_queue: String,
9068        payload_codec: String,
9069        resume_signal: Option<ResumeSignal>,
9070    ) -> Result<Self> {
9071        Self::new_with_identity(
9072            history,
9073            None,
9074            None,
9075            task_queue,
9076            payload_codec,
9077            resume_signal,
9078        )
9079    }
9080
9081    fn new_with_identity(
9082        history: Vec<HistoryEvent>,
9083        workflow_id: Option<String>,
9084        run_id: Option<String>,
9085        task_queue: String,
9086        payload_codec: String,
9087        resume_signal: Option<ResumeSignal>,
9088    ) -> Result<Self> {
9089        let recorded_commands = recorded_commands(
9090            &history,
9091            &payload_codec,
9092            WorkflowIdentity {
9093                workflow_id: workflow_id.clone(),
9094                run_id: run_id.clone(),
9095            },
9096        )?;
9097        let selection_markers = recorded_selection_markers(&history)?;
9098        let cancelled_selection_members = recorded_selection_cancellations(&history)?;
9099        let recorded_continue_as_new = history
9100            .iter()
9101            .filter(|event| event.event_type == "WorkflowContinuedAsNew")
9102            .collect::<Vec<_>>();
9103        if recorded_continue_as_new.len() > 1 {
9104            return Err(invalid_recorded_history(
9105                "duplicate_continue_as_new_transition",
9106                recorded_continue_as_new
9107                    .last()
9108                    .and_then(|event| durable_event_sequence(event))
9109                    .unwrap_or(0),
9110                "one WorkflowContinuedAsNew event",
9111                &format!(
9112                    "{} WorkflowContinuedAsNew events",
9113                    recorded_continue_as_new.len()
9114                ),
9115                "workflow history records one continue-as-new transition more than once",
9116            ));
9117        }
9118        let recorded_continue_as_new_sequence = recorded_continue_as_new
9119            .first()
9120            .map(|event| {
9121                durable_event_sequence(event).ok_or_else(|| {
9122                    Error::NonDeterministicReplay(ReplayFailure::new(
9123                        "continue_as_new_sequence_missing",
9124                        None,
9125                        Some("recorded transition sequence".to_string()),
9126                        Some("missing sequence".to_string()),
9127                        "WorkflowContinuedAsNew history is missing its recorded sequence",
9128                    ))
9129                })
9130            })
9131            .transpose()?;
9132        let mut message_stream_cursors = HashMap::new();
9133        for event in &history {
9134            if !matches!(
9135                event.event_type.as_str(),
9136                "SignalReceived" | "SignalApplied"
9137            ) || event.payload.get("signal_name").and_then(Value::as_str)
9138                != Some(MESSAGE_STREAM_SIGNAL)
9139            {
9140                continue;
9141            }
9142            let arguments = decode_signal_event_arguments(event, &payload_codec)?;
9143            if arguments.len() != 1 {
9144                continue;
9145            }
9146            let envelope = arguments[0].clone().into_json()?;
9147            let Some(envelope) = envelope.as_object() else {
9148                continue;
9149            };
9150            if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_CURSOR_SCHEMA)
9151            {
9152                continue;
9153            }
9154            let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9155                continue;
9156            };
9157            let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9158            else {
9159                continue;
9160            };
9161            let cursor = message_stream_cursors
9162                .entry(stream_name.to_string())
9163                .or_insert(0);
9164            *cursor = (*cursor).max(through_position);
9165        }
9166        let event_count = u64::try_from(history.len()).unwrap_or(u64::MAX);
9167        let cancel_requested = history.iter().any(|event| {
9168            matches!(
9169                event.event_type.as_str(),
9170                "WorkflowCancellationRequested" | "WorkflowCancelRequested"
9171            )
9172        });
9173        Ok(Self {
9174            workflow_command_identity: String::new(),
9175            workflow_stream_command_counter: 0,
9176            workflow_id,
9177            run_id,
9178            task_queue,
9179            payload_codec,
9180            history_events: Arc::new(history),
9181            history_budget: WorkflowHistoryBudget {
9182                event_count,
9183                ..WorkflowHistoryBudget::default()
9184            },
9185            cancel_requested,
9186            resume_signal,
9187            recorded_commands,
9188            selection_markers,
9189            selection_marker_cursor: 0,
9190            cancelled_selection_members,
9191            recorded_continue_as_new_sequence,
9192            continue_as_new_consumed: false,
9193            command_cursor: 0,
9194            condition_wait_occurrence_counter: 0,
9195            matched_recorded_pending: false,
9196            version_markers: HashMap::new(),
9197            commands: Vec::new(),
9198            message_stream_messages: HashMap::new(),
9199            message_stream_cursors,
9200            message_stream_waits: HashMap::new(),
9201        })
9202    }
9203}
9204
9205enum MessageStreamDelivery {
9206    Message(MessageStreamMessage),
9207    Cursor {
9208        stream_name: String,
9209        through_position: u64,
9210    },
9211}
9212
9213fn decode_message_stream_delivery(arguments: Vec<Value>) -> Result<Option<MessageStreamDelivery>> {
9214    if arguments.len() != 1 {
9215        return Ok(None);
9216    }
9217    let envelope = arguments
9218        .into_iter()
9219        .next()
9220        .expect("one argument was checked");
9221    let Some(envelope) = envelope.as_object() else {
9222        return Ok(None);
9223    };
9224    let Some(stream_name) = envelope.get("stream_name").and_then(Value::as_str) else {
9225        return Ok(None);
9226    };
9227    if envelope.get("schema").and_then(Value::as_str) == Some(MESSAGE_STREAM_CURSOR_SCHEMA) {
9228        let Some(through_position) = envelope.get("through_position").and_then(Value::as_u64)
9229        else {
9230            return Ok(None);
9231        };
9232        return Ok(Some(MessageStreamDelivery::Cursor {
9233            stream_name: stream_name.to_string(),
9234            through_position,
9235        }));
9236    }
9237    if envelope.get("schema").and_then(Value::as_str) != Some(MESSAGE_STREAM_SCHEMA) {
9238        return Ok(None);
9239    }
9240    let Some(message_id) = envelope.get("message_id").and_then(Value::as_str) else {
9241        return Ok(None);
9242    };
9243    let Some(position) = envelope
9244        .get("position")
9245        .and_then(Value::as_u64)
9246        .filter(|value| *value > 0)
9247    else {
9248        return Ok(None);
9249    };
9250    let Some(payload_envelope) = envelope.get("payload_envelope") else {
9251        return Ok(None);
9252    };
9253    let Ok(payload_envelope) = serde_json::from_value::<PayloadEnvelope>(payload_envelope.clone())
9254    else {
9255        return Ok(None);
9256    };
9257    let decoded = decode_avro_value(&payload_envelope)?;
9258    let AvroValue::Array(values) = decoded else {
9259        return Ok(None);
9260    };
9261    Ok(Some(MessageStreamDelivery::Message(MessageStreamMessage {
9262        stream_name: stream_name.to_string(),
9263        message_id: message_id.to_string(),
9264        position,
9265        arguments: values,
9266    })))
9267}
9268
9269#[derive(Clone, Debug)]
9270enum RecordedCommand {
9271    Activity {
9272        sequence: u64,
9273        activity_type: Option<String>,
9274        options: Option<RecordedActivityOptions>,
9275        outcome: Option<ActivityOutcome>,
9276        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9277    },
9278    Timer {
9279        sequence: u64,
9280        delay_seconds: u64,
9281        fired: bool,
9282        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9283    },
9284    ChildWorkflow {
9285        sequence: u64,
9286        workflow_type: Option<String>,
9287        outcome: Option<ChildWorkflowOutcome>,
9288        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9289    },
9290    SignalWait {
9291        sequence: u64,
9292        signal_name: String,
9293        value: Option<Vec<AvroValue>>,
9294        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9295    },
9296    ConditionWait {
9297        sequence: u64,
9298        occurrence_id: String,
9299        condition_key: Option<String>,
9300        predicate_identity: String,
9301        timeout_seconds: Option<u64>,
9302        result: Option<ConditionWaitResult>,
9303        parallel_group_path: Option<Vec<ParallelGroupMetadata>>,
9304    },
9305    SearchAttributes {
9306        sequence: u64,
9307        attributes: Value,
9308        attribute_types: RecordedSnapshotValue<BTreeMap<String, String>>,
9309    },
9310    SideEffect {
9311        sequence: u64,
9312        value: AvroValue,
9313    },
9314    VersionMarker {
9315        sequence: u64,
9316        change_id: String,
9317        version: i32,
9318    },
9319    Memo {
9320        sequence: u64,
9321        entries: AvroValue,
9322    },
9323}
9324
9325#[derive(Clone, Debug, PartialEq, Eq)]
9326struct SelectionMarker {
9327    selection_group_id: String,
9328    selection_group_base_sequence: u64,
9329    selection_group_size: usize,
9330    member_key: SelectionKey,
9331    member_index: usize,
9332    member_base_sequence: u64,
9333    member_size: usize,
9334    operation_kind: String,
9335    operation_identity: String,
9336    outcome: String,
9337    resolution_event_id: String,
9338    resolution_event_type: String,
9339}
9340
9341#[derive(Clone, Debug, PartialEq, Eq)]
9342struct SelectionCancellation {
9343    selection_group_id: String,
9344    member_key: SelectionKey,
9345    member_index: usize,
9346    member_base_sequence: u64,
9347    member_size: usize,
9348    operation_kind: String,
9349    operation_identity: String,
9350}
9351
9352fn recorded_selection_markers(events: &[HistoryEvent]) -> Result<Vec<SelectionMarker>> {
9353    let mut markers: Vec<SelectionMarker> = Vec::new();
9354    for event in events
9355        .iter()
9356        .filter(|event| event.event_type == "SelectionResolved")
9357    {
9358        let payload = &event.payload;
9359        let base_sequence = required_selection_u64(payload, "selection_group_base_sequence")?;
9360        let group_size = required_selection_usize(payload, "selection_group_size")?;
9361        let member_base_sequence = required_selection_u64(payload, "member_base_sequence")?;
9362        let member_size = required_selection_usize(payload, "member_size")?;
9363        let member_index = required_selection_usize_allow_zero(payload, "member_index")?;
9364        let group_id = payload_string(payload, "selection_group_id").ok_or_else(|| {
9365            invalid_recorded_history(
9366                "selection_marker_invalid",
9367                base_sequence,
9368                "non-empty selection_group_id",
9369                &payload.to_string(),
9370                "selection winner history is missing its durable group identity",
9371            )
9372        })?;
9373        let expected_group_id = format!("select-calls:{base_sequence}:{group_size}");
9374        if group_id != expected_group_id {
9375            return Err(invalid_recorded_history(
9376                "selection_marker_invalid",
9377                base_sequence,
9378                &expected_group_id,
9379                &group_id,
9380                "selection winner history contains an incompatible group identity",
9381            ));
9382        }
9383        let group_end = base_sequence
9384            .checked_add(u64::try_from(group_size).unwrap_or(u64::MAX))
9385            .unwrap_or(u64::MAX);
9386        let member_end = member_base_sequence
9387            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
9388            .unwrap_or(u64::MAX);
9389        if member_index >= group_size
9390            || member_base_sequence < base_sequence
9391            || member_end > group_end
9392        {
9393            return Err(invalid_recorded_history(
9394                "selection_marker_invalid",
9395                base_sequence,
9396                "winner member within selection group bounds",
9397                &payload.to_string(),
9398                "selection winner history contains an invalid member range",
9399            ));
9400        }
9401        let operation_kind = payload_string(payload, "operation_kind").ok_or_else(|| {
9402            invalid_recorded_history(
9403                "selection_marker_invalid",
9404                base_sequence,
9405                "selection operation kind",
9406                &payload.to_string(),
9407                "selection winner history is missing its operation kind",
9408            )
9409        })?;
9410        if !matches!(
9411            operation_kind.as_str(),
9412            "activity" | "child" | "timer" | "signal" | "condition" | "group"
9413        ) {
9414            return Err(invalid_recorded_history(
9415                "selection_marker_invalid",
9416                base_sequence,
9417                "activity, child, timer, signal, condition, or group",
9418                &operation_kind,
9419                "selection winner history contains an unsupported operation kind",
9420            ));
9421        }
9422        let operation_identity =
9423            payload_string(payload, "operation_identity").ok_or_else(|| {
9424                invalid_recorded_history(
9425                    "selection_marker_invalid",
9426                    base_sequence,
9427                    "non-empty operation identity",
9428                    &payload.to_string(),
9429                    "selection winner history is missing its durable operation identity",
9430                )
9431            })?;
9432        let outcome = payload_string(payload, "outcome").ok_or_else(|| {
9433            invalid_recorded_history(
9434                "selection_marker_invalid",
9435                base_sequence,
9436                "completed or failed selection outcome",
9437                &payload.to_string(),
9438                "selection winner history is missing its outcome",
9439            )
9440        })?;
9441        if !matches!(outcome.as_str(), "completed" | "failed") {
9442            return Err(invalid_recorded_history(
9443                "selection_marker_invalid",
9444                base_sequence,
9445                "completed or failed selection outcome",
9446                &outcome,
9447                "selection winner history contains an unsupported outcome",
9448            ));
9449        }
9450        let marker = SelectionMarker {
9451            selection_group_id: group_id,
9452            selection_group_base_sequence: base_sequence,
9453            selection_group_size: group_size,
9454            member_key: selection_key_from_value(payload.get("member_key"), base_sequence)?,
9455            member_index,
9456            member_base_sequence,
9457            member_size,
9458            operation_kind,
9459            operation_identity,
9460            outcome,
9461            resolution_event_id: payload_string(payload, "resolution_event_id").ok_or_else(
9462                || {
9463                    invalid_recorded_history(
9464                        "selection_marker_invalid",
9465                        base_sequence,
9466                        "durable resolution_event_id",
9467                        &payload.to_string(),
9468                        "selection winner history is missing its terminal event identity",
9469                    )
9470                },
9471            )?,
9472            resolution_event_type: payload_string(payload, "resolution_event_type").ok_or_else(
9473                || {
9474                    invalid_recorded_history(
9475                        "selection_marker_invalid",
9476                        base_sequence,
9477                        "durable resolution_event_type",
9478                        &payload.to_string(),
9479                        "selection winner history is missing its terminal event type",
9480                    )
9481                },
9482            )?,
9483        };
9484        if let Some(existing) = markers
9485            .iter()
9486            .find(|existing| existing.selection_group_id == marker.selection_group_id)
9487        {
9488            if existing != &marker {
9489                return Err(invalid_recorded_history(
9490                    "selection_marker_conflict",
9491                    base_sequence,
9492                    &format!("one winner for {}", marker.selection_group_id),
9493                    &payload.to_string(),
9494                    "selection history records conflicting winners for one durable group",
9495                ));
9496            }
9497            continue;
9498        }
9499        markers.push(marker);
9500    }
9501    Ok(markers)
9502}
9503
9504fn recorded_selection_cancellations(events: &[HistoryEvent]) -> Result<Vec<SelectionCancellation>> {
9505    let mut cancelled: Vec<SelectionCancellation> = Vec::new();
9506    for event in events
9507        .iter()
9508        .filter(|event| event.event_type == "SelectionOperationCancelled")
9509    {
9510        let group_id = payload_string(&event.payload, "selection_group_id").ok_or_else(|| {
9511            invalid_recorded_history(
9512                "selection_cancellation_invalid",
9513                0,
9514                "non-empty selection_group_id",
9515                &event.payload.to_string(),
9516                "selection cancellation history is missing its group identity",
9517            )
9518        })?;
9519        let member_base_sequence = required_selection_u64(&event.payload, "member_base_sequence")?;
9520        let marker = SelectionCancellation {
9521            selection_group_id: group_id,
9522            member_key: selection_key_from_value(
9523                event.payload.get("member_key"),
9524                member_base_sequence,
9525            )?,
9526            member_index: required_selection_usize_allow_zero(&event.payload, "member_index")?,
9527            member_base_sequence,
9528            member_size: required_selection_usize(&event.payload, "member_size")?,
9529            operation_kind: payload_string(&event.payload, "operation_kind").ok_or_else(|| {
9530                invalid_recorded_history(
9531                    "selection_cancellation_invalid",
9532                    member_base_sequence,
9533                    "selection operation kind",
9534                    &event.payload.to_string(),
9535                    "selection cancellation is missing its operation kind",
9536                )
9537            })?,
9538            operation_identity: payload_string(&event.payload, "operation_identity").ok_or_else(
9539                || {
9540                    invalid_recorded_history(
9541                        "selection_cancellation_invalid",
9542                        member_base_sequence,
9543                        "selection operation identity",
9544                        &event.payload.to_string(),
9545                        "selection cancellation is missing its operation identity",
9546                    )
9547                },
9548            )?,
9549        };
9550        if let Some(existing) = cancelled.iter().find(|recorded| {
9551            recorded.selection_group_id == marker.selection_group_id
9552                && recorded.member_base_sequence == marker.member_base_sequence
9553        }) {
9554            if existing != &marker {
9555                return Err(invalid_recorded_history(
9556                    "selection_cancellation_conflict",
9557                    member_base_sequence,
9558                    "one stable SelectionOperationCancelled marker",
9559                    &event.payload.to_string(),
9560                    "selection cancellation history contains conflicting member metadata",
9561                ));
9562            }
9563        } else {
9564            cancelled.push(marker);
9565        }
9566    }
9567    Ok(cancelled)
9568}
9569
9570fn required_selection_u64(payload: &Value, field: &str) -> Result<u64> {
9571    payload
9572        .get(field)
9573        .and_then(value_as_u64)
9574        .filter(|value| *value > 0)
9575        .ok_or_else(|| {
9576            invalid_recorded_history(
9577                "selection_marker_invalid",
9578                0,
9579                &format!("positive integer {field}"),
9580                &payload.to_string(),
9581                "selection history contains invalid durable identity metadata",
9582            )
9583        })
9584}
9585
9586fn required_selection_usize(payload: &Value, field: &str) -> Result<usize> {
9587    required_selection_usize_allow_zero(payload, field).and_then(|value| {
9588        if value > 0 {
9589            Ok(value)
9590        } else {
9591            Err(invalid_recorded_history(
9592                "selection_marker_invalid",
9593                0,
9594                &format!("positive integer {field}"),
9595                &payload.to_string(),
9596                "selection history contains invalid durable identity metadata",
9597            ))
9598        }
9599    })
9600}
9601
9602fn required_selection_usize_allow_zero(payload: &Value, field: &str) -> Result<usize> {
9603    payload
9604        .get(field)
9605        .and_then(value_as_u64)
9606        .and_then(|value| usize::try_from(value).ok())
9607        .ok_or_else(|| {
9608            invalid_recorded_history(
9609                "selection_marker_invalid",
9610                0,
9611                &format!("non-negative integer {field}"),
9612                &payload.to_string(),
9613                "selection history contains invalid durable identity metadata",
9614            )
9615        })
9616}
9617
9618#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9619struct RecordedActivityOptions {
9620    task_queue: RecordedSnapshotValue<Option<String>>,
9621    execution_mode: RecordedSnapshotValue<Option<String>>,
9622    retry_policy: ActivityRetrySnapshot,
9623}
9624
9625#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9626enum RecordedSnapshotValue<T> {
9627    /// Older history did not persist this field, so it cannot constrain replay.
9628    Unknown,
9629    Known(T),
9630}
9631
9632impl<T: PartialEq> RecordedSnapshotValue<T> {
9633    fn matches_current(&self, current: &Self) -> bool {
9634        match self {
9635            Self::Unknown => true,
9636            Self::Known(recorded) => matches!(current, Self::Known(value) if value == recorded),
9637        }
9638    }
9639}
9640
9641#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
9642struct ActivityRetrySnapshot {
9643    snapshot_version: RecordedSnapshotValue<Option<u64>>,
9644    max_attempts: RecordedSnapshotValue<Option<u64>>,
9645    backoff_seconds: RecordedSnapshotValue<Vec<u64>>,
9646    start_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9647    schedule_to_start_timeout: RecordedSnapshotValue<Option<u64>>,
9648    schedule_to_close_timeout: RecordedSnapshotValue<Option<u64>>,
9649    heartbeat_timeout: RecordedSnapshotValue<Option<u64>>,
9650    non_retryable_error_types: RecordedSnapshotValue<Vec<String>>,
9651}
9652
9653impl ActivityRetrySnapshot {
9654    fn matches_current(&self, current: &Self) -> bool {
9655        self.snapshot_version
9656            .matches_current(&current.snapshot_version)
9657            && self.max_attempts.matches_current(&current.max_attempts)
9658            && self
9659                .backoff_seconds
9660                .matches_current(&current.backoff_seconds)
9661            && self
9662                .start_to_close_timeout
9663                .matches_current(&current.start_to_close_timeout)
9664            && self
9665                .schedule_to_start_timeout
9666                .matches_current(&current.schedule_to_start_timeout)
9667            && self
9668                .schedule_to_close_timeout
9669                .matches_current(&current.schedule_to_close_timeout)
9670            && self
9671                .heartbeat_timeout
9672                .matches_current(&current.heartbeat_timeout)
9673            && self
9674                .non_retryable_error_types
9675                .matches_current(&current.non_retryable_error_types)
9676    }
9677}
9678
9679fn recorded_optional_u64(
9680    object: Option<&serde_json::Map<String, Value>>,
9681    field: &str,
9682) -> RecordedSnapshotValue<Option<u64>> {
9683    match object.and_then(|object| object.get(field)) {
9684        None => RecordedSnapshotValue::Unknown,
9685        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9686        Some(value) => RecordedSnapshotValue::Known(value_as_u64(value)),
9687    }
9688}
9689
9690fn recorded_optional_string(
9691    object: &serde_json::Map<String, Value>,
9692    field: &str,
9693) -> RecordedSnapshotValue<Option<String>> {
9694    match object.get(field) {
9695        None => RecordedSnapshotValue::Unknown,
9696        Some(Value::Null) => RecordedSnapshotValue::Known(None),
9697        Some(value) => RecordedSnapshotValue::Known(value.as_str().map(str::to_string)),
9698    }
9699}
9700
9701fn recorded_activity_retry_snapshot(policy: Option<&Value>) -> ActivityRetrySnapshot {
9702    let policy = policy.and_then(Value::as_object);
9703    let backoff_seconds = policy
9704        .and_then(|policy| policy.get("backoff_seconds"))
9705        .and_then(Value::as_array)
9706        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9707        .map_or(RecordedSnapshotValue::Unknown, RecordedSnapshotValue::Known);
9708    let mut non_retryable_error_types = Vec::new();
9709    for error_type in policy
9710        .and_then(|policy| policy.get("non_retryable_error_types"))
9711        .and_then(Value::as_array)
9712        .into_iter()
9713        .flatten()
9714        .filter_map(Value::as_str)
9715        .map(str::trim)
9716        .filter(|error_type| !error_type.is_empty())
9717    {
9718        if !non_retryable_error_types
9719            .iter()
9720            .any(|recorded| recorded == error_type)
9721        {
9722            non_retryable_error_types.push(error_type.to_string());
9723        }
9724    }
9725
9726    ActivityRetrySnapshot {
9727        snapshot_version: recorded_optional_u64(policy, "snapshot_version"),
9728        max_attempts: recorded_optional_u64(policy, "max_attempts"),
9729        backoff_seconds,
9730        start_to_close_timeout: recorded_optional_u64(policy, "start_to_close_timeout"),
9731        schedule_to_start_timeout: recorded_optional_u64(policy, "schedule_to_start_timeout"),
9732        schedule_to_close_timeout: recorded_optional_u64(policy, "schedule_to_close_timeout"),
9733        heartbeat_timeout: recorded_optional_u64(policy, "heartbeat_timeout"),
9734        non_retryable_error_types: if policy
9735            .is_some_and(|policy| policy.contains_key("non_retryable_error_types"))
9736        {
9737            RecordedSnapshotValue::Known(non_retryable_error_types)
9738        } else {
9739            RecordedSnapshotValue::Unknown
9740        },
9741    }
9742}
9743
9744fn current_activity_retry_snapshot(options: &ValidatedActivityOptions) -> ActivityRetrySnapshot {
9745    let policy = options.retry_policy.as_ref();
9746    let max_attempts = match policy.and_then(|policy| policy.get("max_attempts")) {
9747        Some(Value::Null) => None,
9748        Some(value) => value_as_u64(value),
9749        None => Some(1),
9750    };
9751    let backoff_seconds = policy
9752        .and_then(|policy| policy.get("backoff_seconds"))
9753        .and_then(Value::as_array)
9754        .map(|intervals| intervals.iter().filter_map(value_as_u64).collect())
9755        .unwrap_or_default();
9756    let non_retryable_error_types = policy
9757        .and_then(|policy| policy.get("non_retryable_error_types"))
9758        .and_then(Value::as_array)
9759        .into_iter()
9760        .flatten()
9761        .filter_map(Value::as_str)
9762        .map(str::to_string)
9763        .collect();
9764
9765    ActivityRetrySnapshot {
9766        snapshot_version: RecordedSnapshotValue::Known(Some(1)),
9767        max_attempts: RecordedSnapshotValue::Known(max_attempts),
9768        backoff_seconds: RecordedSnapshotValue::Known(backoff_seconds),
9769        start_to_close_timeout: RecordedSnapshotValue::Known(options.start_to_close_timeout),
9770        schedule_to_start_timeout: RecordedSnapshotValue::Known(options.schedule_to_start_timeout),
9771        schedule_to_close_timeout: RecordedSnapshotValue::Known(options.schedule_to_close_timeout),
9772        heartbeat_timeout: RecordedSnapshotValue::Known(options.heartbeat_timeout),
9773        non_retryable_error_types: RecordedSnapshotValue::Known(non_retryable_error_types),
9774    }
9775}
9776
9777fn activity_options_description(options: &RecordedActivityOptions) -> String {
9778    serde_json::to_string(options).unwrap_or_else(|_| format!("{options:?}"))
9779}
9780
9781impl RecordedCommand {
9782    fn sequence(&self) -> u64 {
9783        match self {
9784            Self::Activity { sequence, .. }
9785            | Self::Timer { sequence, .. }
9786            | Self::ChildWorkflow { sequence, .. }
9787            | Self::SignalWait { sequence, .. }
9788            | Self::ConditionWait { sequence, .. }
9789            | Self::SearchAttributes { sequence, .. }
9790            | Self::SideEffect { sequence, .. }
9791            | Self::VersionMarker { sequence, .. }
9792            | Self::Memo { sequence, .. } => *sequence,
9793        }
9794    }
9795
9796    fn shape(&self) -> &'static str {
9797        match self {
9798            Self::Activity { .. } => "activity",
9799            Self::Timer { .. } => "timer",
9800            Self::ChildWorkflow { .. } => "child workflow",
9801            Self::SignalWait { .. } => "signal wait",
9802            Self::ConditionWait { .. } => "condition wait",
9803            Self::SearchAttributes { .. } => "search-attribute update",
9804            Self::SideEffect { .. } => "side effect",
9805            Self::VersionMarker { .. } => "version marker",
9806            Self::Memo { .. } => "memo upsert",
9807        }
9808    }
9809}
9810
9811fn ensure_version_supported(
9812    change_id: &str,
9813    version: i32,
9814    min_supported: i32,
9815    max_supported: i32,
9816    sequence: u64,
9817) -> Result<()> {
9818    if (min_supported..=max_supported).contains(&version) {
9819        return Ok(());
9820    }
9821    Err(Error::NonDeterministicReplay(ReplayFailure::new(
9822        "version_marker_incompatible_range",
9823        (sequence != 0).then_some(sequence),
9824        Some(format!("{min_supported}..={max_supported}")),
9825        Some(format!("{change_id}:{version}")),
9826        "recorded workflow version is outside the range supported by current code",
9827    )))
9828}
9829
9830#[derive(Clone, Debug)]
9831struct ResumeSignal {
9832    signal_name: String,
9833    arguments: Vec<AvroValue>,
9834}
9835
9836const MAX_PARALLEL_OPERATIONS: usize = 1000;
9837
9838fn parallel_group_prefix(kind: &str) -> &'static str {
9839    match kind {
9840        "activity" => "parallel-activities",
9841        "child" => "parallel-children",
9842        "timer" => "parallel-timers",
9843        _ => "parallel-calls",
9844    }
9845}
9846
9847fn parallel_group_entry(
9848    base_sequence: u64,
9849    size: usize,
9850    index: usize,
9851    kind: &str,
9852) -> ParallelGroupMetadata {
9853    ParallelGroupMetadata {
9854        parallel_group_id: format!("{}:{base_sequence}:{size}", parallel_group_prefix(kind)),
9855        parallel_group_kind: kind.to_string(),
9856        parallel_group_base_sequence: base_sequence,
9857        parallel_group_size: size,
9858        parallel_group_index: index,
9859        parallel_group_mode: None,
9860        selection_member_key: None,
9861        selection_member_index: None,
9862        selection_member_base_sequence: None,
9863        selection_member_size: None,
9864        selection_member_kind: None,
9865    }
9866}
9867
9868struct SelectionMemberMetadata {
9869    key: SelectionKey,
9870    index: usize,
9871    base_sequence: u64,
9872    size: usize,
9873    kind: String,
9874}
9875
9876fn selection_group_entry(
9877    base_sequence: u64,
9878    size: usize,
9879    index: usize,
9880    kind: &str,
9881    member: &SelectionMemberMetadata,
9882) -> ParallelGroupMetadata {
9883    ParallelGroupMetadata {
9884        parallel_group_id: format!("select-calls:{base_sequence}:{size}"),
9885        parallel_group_kind: kind.to_string(),
9886        parallel_group_base_sequence: base_sequence,
9887        parallel_group_size: size,
9888        parallel_group_index: index,
9889        parallel_group_mode: Some("select".to_string()),
9890        selection_member_key: Some(member.key.clone()),
9891        selection_member_index: Some(member.index),
9892        selection_member_base_sequence: Some(member.base_sequence),
9893        selection_member_size: Some(member.size),
9894        selection_member_kind: Some(member.kind.clone()),
9895    }
9896}
9897
9898fn apply_parallel_group_path(
9899    command: &mut serde_json::Map<String, Value>,
9900    path: &[ParallelGroupMetadata],
9901) {
9902    let Some(inner) = path.last() else {
9903        return;
9904    };
9905    command.insert(
9906        "parallel_group_id".to_string(),
9907        json!(inner.parallel_group_id),
9908    );
9909    command.insert(
9910        "parallel_group_kind".to_string(),
9911        json!(inner.parallel_group_kind),
9912    );
9913    command.insert(
9914        "parallel_group_base_sequence".to_string(),
9915        json!(inner.parallel_group_base_sequence),
9916    );
9917    command.insert(
9918        "parallel_group_size".to_string(),
9919        json!(inner.parallel_group_size),
9920    );
9921    command.insert(
9922        "parallel_group_index".to_string(),
9923        json!(inner.parallel_group_index),
9924    );
9925    if let Some(mode) = &inner.parallel_group_mode {
9926        command.insert("parallel_group_mode".to_string(), json!(mode));
9927    }
9928    if let Some(key) = &inner.selection_member_key {
9929        command.insert("selection_member_key".to_string(), json!(key));
9930    }
9931    if let Some(index) = inner.selection_member_index {
9932        command.insert("selection_member_index".to_string(), json!(index));
9933    }
9934    if let Some(base_sequence) = inner.selection_member_base_sequence {
9935        command.insert(
9936            "selection_member_base_sequence".to_string(),
9937            json!(base_sequence),
9938        );
9939    }
9940    if let Some(size) = inner.selection_member_size {
9941        command.insert("selection_member_size".to_string(), json!(size));
9942    }
9943    if let Some(kind) = &inner.selection_member_kind {
9944        command.insert("selection_member_kind".to_string(), json!(kind));
9945    }
9946    command.insert("parallel_group_path".to_string(), json!(path));
9947}
9948
9949fn ensure_parallel_path_matches(
9950    sequence: u64,
9951    recorded: Option<&[ParallelGroupMetadata]>,
9952    expected: &[ParallelGroupMetadata],
9953) -> Result<()> {
9954    match (recorded, expected.is_empty()) {
9955        (None, true) => Ok(()),
9956        (Some(recorded), false) if recorded == expected => Ok(()),
9957        (None, false) => Err(invalid_recorded_history(
9958            "parallel_group_metadata_missing",
9959            sequence,
9960            &serde_json::to_string(expected).unwrap_or_default(),
9961            "<missing>",
9962            "recorded parallel member is missing its durable group path",
9963        )),
9964        (Some(recorded), true) => Err(invalid_recorded_history(
9965            "parallel_group_shape_mismatch",
9966            sequence,
9967            "sequential command",
9968            &serde_json::to_string(recorded).unwrap_or_default(),
9969            "recorded command belonged to a parallel group but current code schedules it sequentially",
9970        )),
9971        (Some(recorded), false) => Err(invalid_recorded_history(
9972            "parallel_group_shape_mismatch",
9973            sequence,
9974            &serde_json::to_string(recorded).unwrap_or_default(),
9975            &serde_json::to_string(expected).unwrap_or_default(),
9976            "recorded parallel-group identity or path changed during replay",
9977        )),
9978    }
9979}
9980
9981#[derive(Clone, Debug)]
9982enum ParallelShape {
9983    Leaf,
9984    Group(Vec<ParallelShape>),
9985}
9986
9987struct ParallelDescriptor {
9988    operation: ParallelOperation,
9989    offset: usize,
9990    member_path: Vec<usize>,
9991    group_path: Vec<ParallelGroupMetadata>,
9992}
9993
9994fn parallel_leaf_count(operations: &[ParallelOperation]) -> usize {
9995    operations
9996        .iter()
9997        .map(|operation| match operation {
9998            ParallelOperation::Group(children) => parallel_leaf_count(children),
9999            _ => 1,
10000        })
10001        .sum()
10002}
10003
10004fn parallel_operation_kind(operation: &ParallelOperation) -> Option<&'static str> {
10005    match operation {
10006        ParallelOperation::Activity { .. } => Some("activity"),
10007        ParallelOperation::ChildWorkflow { .. } => Some("child"),
10008        ParallelOperation::Timer(_) => Some("timer"),
10009        ParallelOperation::Signal(_) => Some("signal"),
10010        ParallelOperation::Condition { .. } => Some("condition"),
10011        ParallelOperation::Group(children) => parallel_group_kind(children),
10012    }
10013}
10014
10015fn parallel_group_kind(operations: &[ParallelOperation]) -> Option<&'static str> {
10016    let mut kind = None;
10017    for operation in operations {
10018        let Some(operation_kind) = parallel_operation_kind(operation) else {
10019            continue;
10020        };
10021        match kind {
10022            None => kind = Some(operation_kind),
10023            Some(current) if current == operation_kind => {}
10024            Some(_) => return Some("mixed"),
10025        }
10026    }
10027    kind
10028}
10029
10030fn validate_parallel_operations(
10031    operations: &[ParallelOperation],
10032    member_path: &mut Vec<usize>,
10033    root: bool,
10034) -> Result<()> {
10035    let leaves = parallel_leaf_count(operations);
10036    if leaves > MAX_PARALLEL_OPERATIONS {
10037        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10038            reason: "fan_out_limit_exceeded",
10039            member_path: member_path.clone(),
10040            message: format!(
10041                "group contains {leaves} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10042            ),
10043        }));
10044    }
10045    if !root && operations.is_empty() {
10046        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10047            reason: "nested_group_empty",
10048            member_path: member_path.clone(),
10049            message: "a nested group must contain at least one durable leaf".to_string(),
10050        }));
10051    }
10052
10053    for (index, operation) in operations.iter().enumerate() {
10054        member_path.push(index);
10055        match operation {
10056            ParallelOperation::Activity {
10057                options, arguments, ..
10058            } => {
10059                options
10060                    .validate()
10061                    .map_err(|error| Error::InvalidActivityOptions(error))?;
10062                if let Err(error) = arguments {
10063                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10064                        reason: "arguments_invalid",
10065                        member_path: member_path.clone(),
10066                        message: error.to_string(),
10067                    }));
10068                }
10069            }
10070            ParallelOperation::ChildWorkflow {
10071                options, arguments, ..
10072            } => {
10073                validate_parallel_child_options(options)?;
10074                if let Err(error) = arguments {
10075                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10076                        reason: "arguments_invalid",
10077                        member_path: member_path.clone(),
10078                        message: error.to_string(),
10079                    }));
10080                }
10081            }
10082            ParallelOperation::Timer(duration)
10083                if duration.as_secs() == u64::MAX && duration.subsec_nanos() > 0 =>
10084            {
10085                return Err(Error::TimerDurationOverflow);
10086            }
10087            ParallelOperation::Timer(_) => {}
10088            ParallelOperation::Signal(signal_name) => {
10089                validate_user_signal_name(signal_name)?;
10090                if signal_name.trim().is_empty() {
10091                    return Err(Error::InvalidParallelGroup(ParallelGroupError {
10092                        reason: "signal_name_empty",
10093                        member_path: member_path.clone(),
10094                        message: "signal wait name must not be empty".to_string(),
10095                    }));
10096                }
10097            }
10098            ParallelOperation::Condition { options, .. } => {
10099                options.validate()?;
10100            }
10101            ParallelOperation::Group(children) => {
10102                validate_parallel_operations(children, member_path, false)?;
10103            }
10104        }
10105        member_path.pop();
10106    }
10107    Ok(())
10108}
10109
10110fn validate_parallel_child_options(options: &ChildWorkflowOptions) -> Result<()> {
10111    if options.task_queue.trim().is_empty() {
10112        return Err(Error::InvalidChildWorkflowOptions(
10113            "task_queue must not be empty".to_string(),
10114        ));
10115    }
10116    for (name, value) in [
10117        (
10118            "execution_timeout_seconds",
10119            options.execution_timeout_seconds,
10120        ),
10121        ("run_timeout_seconds", options.run_timeout_seconds),
10122    ] {
10123        if value == Some(0) {
10124            return Err(Error::InvalidChildWorkflowOptions(format!(
10125                "{name} must be at least 1"
10126            )));
10127        }
10128    }
10129    if options
10130        .retry_policy
10131        .as_ref()
10132        .is_some_and(|policy| policy.max_attempts == Some(0))
10133    {
10134        return Err(Error::InvalidChildWorkflowOptions(
10135            "retry_policy.max_attempts must be at least 1".to_string(),
10136        ));
10137    }
10138    Ok(())
10139}
10140
10141fn parallel_shape(operations: &[ParallelOperation]) -> ParallelShape {
10142    ParallelShape::Group(
10143        operations
10144            .iter()
10145            .map(|operation| match operation {
10146                ParallelOperation::Group(children) => parallel_shape(children),
10147                _ => ParallelShape::Leaf,
10148            })
10149            .collect(),
10150    )
10151}
10152
10153fn parallel_descriptors(
10154    operations: Vec<ParallelOperation>,
10155    base_sequence: u64,
10156) -> Result<Vec<ParallelDescriptor>> {
10157    let size = parallel_leaf_count(&operations);
10158    let kind = parallel_group_kind(&operations).unwrap_or("activity");
10159    let mut descriptors = Vec::with_capacity(size);
10160    let mut cursor = 0;
10161
10162    for (index, operation) in operations.into_iter().enumerate() {
10163        match operation {
10164            ParallelOperation::Group(children) => {
10165                let child_base = base_sequence
10166                    .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10167                    .ok_or(Error::TimerDurationOverflow)?;
10168                for mut descriptor in parallel_descriptors(children, child_base)? {
10169                    let outer_index = cursor + descriptor.offset;
10170                    descriptor.group_path.insert(
10171                        0,
10172                        parallel_group_entry(base_sequence, size, outer_index, kind),
10173                    );
10174                    descriptor.member_path.insert(0, index);
10175                    descriptor.offset = outer_index;
10176                    descriptors.push(descriptor);
10177                }
10178                cursor = descriptors.len();
10179            }
10180            operation => {
10181                descriptors.push(ParallelDescriptor {
10182                    operation,
10183                    offset: cursor,
10184                    member_path: vec![index],
10185                    group_path: vec![parallel_group_entry(base_sequence, size, cursor, kind)],
10186                });
10187                cursor += 1;
10188            }
10189        }
10190    }
10191    Ok(descriptors)
10192}
10193
10194enum ParallelLeafCall {
10195    Activity(ActivityCall),
10196    ChildWorkflow(ChildWorkflowCall),
10197    Timer(TimerCall),
10198    Signal(SignalCall),
10199    Condition(ConditionWaitCall),
10200}
10201
10202fn parallel_leaf_call(
10203    ctx: &WorkflowContext,
10204    operation: ParallelOperation,
10205    parallel_group_path: Vec<ParallelGroupMetadata>,
10206) -> ParallelLeafCall {
10207    match operation {
10208        ParallelOperation::Activity {
10209            activity_type,
10210            options,
10211            arguments,
10212        } => ParallelLeafCall::Activity(ActivityCall {
10213            ctx: ctx.clone(),
10214            activity_type,
10215            options,
10216            args: Some(arguments),
10217            scheduled: false,
10218            parallel_group_path,
10219        }),
10220        ParallelOperation::ChildWorkflow {
10221            workflow_type,
10222            options,
10223            arguments,
10224        } => ParallelLeafCall::ChildWorkflow(ChildWorkflowCall {
10225            ctx: ctx.clone(),
10226            workflow_type,
10227            options,
10228            args: Some(arguments),
10229            scheduled: false,
10230            matched_pending: false,
10231            parallel_group_path,
10232        }),
10233        ParallelOperation::Timer(duration) => {
10234            let delay_seconds = duration
10235                .as_secs()
10236                .checked_add(u64::from(duration.subsec_nanos() > 0));
10237            ParallelLeafCall::Timer(TimerCall {
10238                ctx: ctx.clone(),
10239                delay_seconds,
10240                scheduled: false,
10241                matched_pending: false,
10242                parallel_group_path,
10243            })
10244        }
10245        ParallelOperation::Signal(signal_name) => ParallelLeafCall::Signal(SignalCall {
10246            ctx: ctx.clone(),
10247            signal_name,
10248            runtime_reserved_allowed: false,
10249            opened_wait: false,
10250            matched_pending: false,
10251            parallel_group_path,
10252        }),
10253        ParallelOperation::Condition { options, predicate } => {
10254            ParallelLeafCall::Condition(ConditionWaitCall {
10255                ctx: ctx.clone(),
10256                options,
10257                predicate,
10258                occurrence_id: None,
10259                opened_wait: false,
10260                parallel_group_path,
10261            })
10262        }
10263        ParallelOperation::Group(_) => {
10264            unreachable!("parallel descriptors contain only durable leaves")
10265        }
10266    }
10267}
10268
10269impl ParallelLeafCall {
10270    fn poll_avro_value(&mut self, cx: &mut TaskContext<'_>) -> Poll<Result<ParallelAvroResult>> {
10271        match self {
10272            Self::Activity(call) => Pin::new(call)
10273                .poll_avro_value(cx)
10274                .map_ok(ParallelAvroResult::Activity),
10275            Self::ChildWorkflow(call) => Pin::new(call)
10276                .poll_avro_value(cx)
10277                .map_ok(ParallelAvroResult::ChildWorkflow),
10278            Self::Timer(call) => Pin::new(call)
10279                .poll(cx)
10280                .map_ok(|()| ParallelAvroResult::Timer),
10281            Self::Signal(call) => Pin::new(call)
10282                .poll_avro_value(cx)
10283                .map_ok(ParallelAvroResult::Signal),
10284            Self::Condition(call) => Pin::new(call)
10285                .poll(cx)
10286                .map_ok(ParallelAvroResult::Condition),
10287        }
10288    }
10289}
10290
10291struct ParallelLeaf {
10292    call: ParallelLeafCall,
10293    member_path: Vec<usize>,
10294    group_path: Vec<ParallelGroupMetadata>,
10295    result: Option<ParallelAvroResult>,
10296}
10297
10298/// Future returned by [`WorkflowContext::parallel`].
10299pub struct ParallelCall {
10300    ctx: WorkflowContext,
10301    operations: Option<Vec<ParallelOperation>>,
10302    shape: Option<ParallelShape>,
10303    leaves: Vec<ParallelLeaf>,
10304}
10305
10306impl ParallelCall {
10307    fn new(ctx: WorkflowContext, operations: Vec<ParallelOperation>) -> Self {
10308        Self {
10309            ctx,
10310            operations: Some(operations),
10311            shape: None,
10312            leaves: Vec::new(),
10313        }
10314    }
10315
10316    fn initialize(&mut self) -> Result<()> {
10317        let operations = self.operations.take().unwrap_or_default();
10318        validate_parallel_operations(&operations, &mut Vec::new(), true)?;
10319        self.shape = Some(parallel_shape(&operations));
10320        if operations.is_empty() {
10321            return Ok(());
10322        }
10323
10324        let base_sequence = {
10325            let state = self
10326                .ctx
10327                .state
10328                .lock()
10329                .map_err(|_| Error::WorkflowStatePoisoned)?;
10330            if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10331                recorded.sequence()
10332            } else {
10333                let last = state
10334                    .recorded_commands
10335                    .last()
10336                    .map(RecordedCommand::sequence)
10337                    .unwrap_or(0);
10338                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10339                    .and_then(|sequence| sequence.checked_add(1))
10340                    .ok_or_else(|| {
10341                        Error::InvalidParallelGroup(ParallelGroupError {
10342                            reason: "sequence_overflow",
10343                            member_path: Vec::new(),
10344                            message: "parallel group sequence identity overflowed u64".to_string(),
10345                        })
10346                    })?
10347            }
10348        };
10349
10350        self.leaves = parallel_descriptors(operations, base_sequence)?
10351            .into_iter()
10352            .map(|descriptor| {
10353                let call = parallel_leaf_call(
10354                    &self.ctx,
10355                    descriptor.operation,
10356                    descriptor.group_path.clone(),
10357                );
10358                ParallelLeaf {
10359                    call,
10360                    member_path: descriptor.member_path,
10361                    group_path: descriptor.group_path,
10362                    result: None,
10363                }
10364            })
10365            .collect();
10366        Ok(())
10367    }
10368
10369    fn poll_avro_value(
10370        mut self: Pin<&mut Self>,
10371        cx: &mut TaskContext<'_>,
10372    ) -> Poll<Result<Vec<ParallelAvroResult>>> {
10373        if self.operations.is_some() {
10374            if let Err(error) = self.initialize() {
10375                return Poll::Ready(Err(error));
10376            }
10377        }
10378        if self.leaves.is_empty() {
10379            return Poll::Ready(Ok(Vec::new()));
10380        }
10381
10382        let mut failures = Vec::new();
10383        let mut pending = false;
10384        for (index, leaf) in self.leaves.iter_mut().enumerate() {
10385            if leaf.result.is_some() {
10386                continue;
10387            }
10388            match leaf.call.poll_avro_value(cx) {
10389                Poll::Ready(Ok(result)) => leaf.result = Some(result),
10390                Poll::Ready(Err(error)) => failures.push((index, error)),
10391                Poll::Pending => pending = true,
10392            }
10393        }
10394
10395        if !failures.is_empty() {
10396            if let Some(position) = failures
10397                .iter()
10398                .position(|(_, error)| workflow_task_integrity_error(error))
10399            {
10400                return Poll::Ready(Err(failures.remove(position).1));
10401            }
10402            failures.sort_by_key(|(index, _)| *index);
10403            let (failed_index, cause) = failures.remove(0);
10404            let failed = &self.leaves[failed_index];
10405            let completed = self
10406                .leaves
10407                .iter()
10408                .filter_map(|leaf| {
10409                    leaf.result
10410                        .clone()
10411                        .and_then(|result| result.into_json_result().ok())
10412                        .map(|result| ParallelCompletion {
10413                            member_path: leaf.member_path.clone(),
10414                            result,
10415                        })
10416                })
10417                .collect();
10418            let group_id = failed
10419                .group_path
10420                .first()
10421                .map(|entry| entry.parallel_group_id.clone())
10422                .unwrap_or_default();
10423            return Poll::Ready(Err(Error::ParallelFailed(ParallelFailure {
10424                group_id,
10425                member_path: failed.member_path.clone(),
10426                group_path: failed.group_path.clone(),
10427                completed,
10428                cause: Box::new(cause),
10429            })));
10430        }
10431        if pending {
10432            return Poll::Pending;
10433        }
10434
10435        let mut flat_results = self
10436            .leaves
10437            .iter_mut()
10438            .map(|leaf| leaf.result.take().expect("completed parallel leaf"))
10439            .collect::<Vec<_>>()
10440            .into_iter();
10441        let results = parallel_results_for_shape(
10442            self.shape.as_ref().expect("initialized parallel shape"),
10443            &mut flat_results,
10444        );
10445        Poll::Ready(Ok(match results {
10446            ParallelAvroResult::Group(results) => results,
10447            ParallelAvroResult::Activity(_)
10448            | ParallelAvroResult::ChildWorkflow(_)
10449            | ParallelAvroResult::Timer
10450            | ParallelAvroResult::Signal(_)
10451            | ParallelAvroResult::Condition(_) => {
10452                unreachable!("root parallel shape is a group")
10453            }
10454        }))
10455    }
10456}
10457
10458fn parallel_results_for_shape(
10459    shape: &ParallelShape,
10460    flat_results: &mut impl Iterator<Item = ParallelAvroResult>,
10461) -> ParallelAvroResult {
10462    match shape {
10463        ParallelShape::Leaf => flat_results.next().expect("one result per parallel leaf"),
10464        ParallelShape::Group(children) => ParallelAvroResult::Group(
10465            children
10466                .iter()
10467                .map(|child| parallel_results_for_shape(child, flat_results))
10468                .collect(),
10469        ),
10470    }
10471}
10472
10473impl Future for ParallelCall {
10474    type Output = Result<Vec<ParallelResult>>;
10475
10476    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10477        self.poll_avro_value(cx)
10478            .map_ok(|results| {
10479                results
10480                    .into_iter()
10481                    .map(ParallelAvroResult::into_json_result)
10482                    .collect::<Result<Vec<_>>>()
10483            })
10484            .map_ok(|result| result)
10485            .flatten_result()
10486    }
10487}
10488
10489#[derive(Clone, Debug)]
10490struct SelectionMemberPlan {
10491    key: SelectionKey,
10492    index: usize,
10493    base_sequence: u64,
10494    size: usize,
10495    kind: String,
10496    shape: ParallelShape,
10497    leaf_start: usize,
10498}
10499
10500fn selection_operation_kind(operation: &ParallelOperation) -> &'static str {
10501    match operation {
10502        ParallelOperation::Activity { .. } => "activity",
10503        ParallelOperation::ChildWorkflow { .. } => "child",
10504        ParallelOperation::Timer(_) => "timer",
10505        ParallelOperation::Signal(_) => "signal",
10506        ParallelOperation::Condition { .. } => "condition",
10507        ParallelOperation::Group(_) => "group",
10508    }
10509}
10510
10511fn selection_operation_shape(operation: &ParallelOperation) -> ParallelShape {
10512    match operation {
10513        ParallelOperation::Group(children) => parallel_shape(children),
10514        _ => ParallelShape::Leaf,
10515    }
10516}
10517
10518fn selection_descriptors(
10519    operations: Vec<(SelectionKey, ParallelOperation)>,
10520    base_sequence: u64,
10521) -> Result<(Vec<ParallelDescriptor>, Vec<SelectionMemberPlan>)> {
10522    if operations.is_empty() {
10523        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10524            reason: "selection_empty",
10525            member_path: Vec::new(),
10526            message: "durable selection requires at least one operation".to_string(),
10527        }));
10528    }
10529    let operation_refs = operations
10530        .iter()
10531        .map(|(_, operation)| operation)
10532        .collect::<Vec<_>>();
10533    let total_size = operation_refs
10534        .iter()
10535        .map(|operation| match operation {
10536            ParallelOperation::Group(children) => parallel_leaf_count(children),
10537            _ => 1,
10538        })
10539        .sum::<usize>();
10540    if total_size > MAX_PARALLEL_OPERATIONS {
10541        return Err(Error::InvalidParallelGroup(ParallelGroupError {
10542            reason: "fan_out_limit_exceeded",
10543            member_path: Vec::new(),
10544            message: format!(
10545                "selection contains {total_size} durable leaves; the limit is {MAX_PARALLEL_OPERATIONS}"
10546            ),
10547        }));
10548    }
10549    let group_kind = {
10550        let mut kind = None;
10551        for operation in &operation_refs {
10552            let operation_kind = parallel_operation_kind(operation).unwrap_or("mixed");
10553            match kind {
10554                None => kind = Some(operation_kind),
10555                Some(current) if current == operation_kind => {}
10556                Some(_) => {
10557                    kind = Some("mixed");
10558                    break;
10559                }
10560            }
10561        }
10562        kind.unwrap_or("mixed")
10563    };
10564
10565    let mut descriptors = Vec::with_capacity(total_size);
10566    let mut members = Vec::with_capacity(operations.len());
10567    let mut cursor = 0usize;
10568    let mut seen_keys: Vec<SelectionKey> = Vec::new();
10569    for (member_index, (key, operation)) in operations.into_iter().enumerate() {
10570        if matches!(&key, SelectionKey::Name(value) if value.is_empty()) {
10571            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10572                reason: "selection_key_invalid",
10573                member_path: vec![member_index],
10574                message: "selection member keys must be non-empty strings or non-negative integers"
10575                    .to_string(),
10576            }));
10577        }
10578        if seen_keys.contains(&key) {
10579            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10580                reason: "selection_key_duplicate",
10581                member_path: vec![member_index],
10582                message: format!("selection member key {key:?} is duplicated"),
10583            }));
10584        }
10585        seen_keys.push(key.clone());
10586        let member_size = match &operation {
10587            ParallelOperation::Group(children) => parallel_leaf_count(children),
10588            _ => 1,
10589        };
10590        if member_size == 0 {
10591            return Err(Error::InvalidParallelGroup(ParallelGroupError {
10592                reason: "selection_member_empty",
10593                member_path: vec![member_index],
10594                message: "a selection member must contain at least one durable leaf".to_string(),
10595            }));
10596        }
10597        let member_base = base_sequence
10598            .checked_add(u64::try_from(cursor).unwrap_or(u64::MAX))
10599            .ok_or(Error::TimerDurationOverflow)?;
10600        let member_kind = selection_operation_kind(&operation).to_string();
10601        let member_shape = selection_operation_shape(&operation);
10602        let leaf_start = descriptors.len();
10603        match operation {
10604            ParallelOperation::Group(children) => {
10605                validate_parallel_operations(&children, &mut vec![member_index], false)?;
10606                for mut descriptor in parallel_descriptors(children, member_base)? {
10607                    let flat_index = cursor + descriptor.offset;
10608                    descriptor.group_path.insert(
10609                        0,
10610                        selection_group_entry(
10611                            base_sequence,
10612                            total_size,
10613                            flat_index,
10614                            group_kind,
10615                            &SelectionMemberMetadata {
10616                                key: key.clone(),
10617                                index: member_index,
10618                                base_sequence: member_base,
10619                                size: member_size,
10620                                kind: member_kind.clone(),
10621                            },
10622                        ),
10623                    );
10624                    descriptor.member_path.insert(0, member_index);
10625                    descriptor.offset = flat_index;
10626                    descriptors.push(descriptor);
10627                }
10628            }
10629            operation => {
10630                validate_parallel_operations(
10631                    std::slice::from_ref(&operation),
10632                    &mut Vec::new(),
10633                    true,
10634                )?;
10635                descriptors.push(ParallelDescriptor {
10636                    operation,
10637                    offset: cursor,
10638                    member_path: vec![member_index],
10639                    group_path: vec![selection_group_entry(
10640                        base_sequence,
10641                        total_size,
10642                        cursor,
10643                        group_kind,
10644                        &SelectionMemberMetadata {
10645                            key: key.clone(),
10646                            index: member_index,
10647                            base_sequence: member_base,
10648                            size: member_size,
10649                            kind: member_kind.clone(),
10650                        },
10651                    )],
10652                });
10653            }
10654        }
10655        members.push(SelectionMemberPlan {
10656            key,
10657            index: member_index,
10658            base_sequence: member_base,
10659            size: member_size,
10660            kind: member_kind,
10661            shape: member_shape,
10662            leaf_start,
10663        });
10664        cursor += member_size;
10665    }
10666    Ok((descriptors, members))
10667}
10668
10669struct SelectionLeaf {
10670    call: ParallelLeafCall,
10671    outcome: Option<Result<ParallelAvroResult>>,
10672}
10673
10674/// Stable reference to one member of a durable selection group.
10675#[derive(Clone)]
10676pub struct DurableOperationHandle {
10677    ctx: WorkflowContext,
10678    pub key: SelectionKey,
10679    pub index: usize,
10680    pub kind: String,
10681    pub identity: String,
10682    pub base_sequence: u64,
10683    pub size: usize,
10684    pub selection_group_id: String,
10685    shape: ParallelShape,
10686}
10687
10688impl std::fmt::Debug for DurableOperationHandle {
10689    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10690        formatter
10691            .debug_struct("DurableOperationHandle")
10692            .field("key", &self.key)
10693            .field("index", &self.index)
10694            .field("kind", &self.kind)
10695            .field("identity", &self.identity)
10696            .field("base_sequence", &self.base_sequence)
10697            .field("size", &self.size)
10698            .field("selection_group_id", &self.selection_group_id)
10699            .finish()
10700    }
10701}
10702
10703impl DurableOperationHandle {
10704    /// Await this member after another member has already won the selection.
10705    pub fn await_result(&self) -> DurableOperationAwaitCall {
10706        DurableOperationAwaitCall {
10707            handle: self.clone(),
10708        }
10709    }
10710
10711    /// Request cancellation without affecting siblings. The future resolves
10712    /// to unit; only `SelectionOperationCancelled` history proves cancellation
10713    /// beat a concurrently committed terminal result.
10714    pub fn cancel(&self) -> CancelDurableOperationCall {
10715        CancelDurableOperationCall {
10716            handle: self.clone(),
10717            emitted: false,
10718        }
10719    }
10720}
10721
10722/// The one winner committed for a durable selection group.
10723#[derive(Debug)]
10724pub struct SelectionResult {
10725    pub key: SelectionKey,
10726    pub index: usize,
10727    pub kind: String,
10728    pub identity: String,
10729    pub value: Option<ParallelResult>,
10730    pub failure: Option<Error>,
10731    pub winner: DurableOperationHandle,
10732    pub handles: Vec<DurableOperationHandle>,
10733}
10734
10735impl SelectionResult {
10736    pub fn succeeded(&self) -> bool {
10737        self.failure.is_none()
10738    }
10739
10740    pub fn handle(&self, key: &SelectionKey) -> Option<&DurableOperationHandle> {
10741        self.handles.iter().find(|handle| &handle.key == key)
10742    }
10743
10744    pub fn remaining(&self) -> Vec<&DurableOperationHandle> {
10745        self.handles
10746            .iter()
10747            .filter(|handle| handle.index != self.index)
10748            .collect()
10749    }
10750
10751    pub fn into_result(self) -> Result<ParallelResult> {
10752        match (self.value, self.failure) {
10753            (Some(value), None) => Ok(value),
10754            (_, Some(error)) => Err(error),
10755            _ => Err(Error::WorkerLoop(
10756                "selection result contained neither a value nor a failure".to_string(),
10757            )),
10758        }
10759    }
10760}
10761
10762/// Future returned by [`WorkflowContext::select`].
10763pub struct SelectCall {
10764    ctx: WorkflowContext,
10765    operations: Option<Vec<(SelectionKey, ParallelOperation)>>,
10766    members: Vec<SelectionMemberPlan>,
10767    leaves: Vec<SelectionLeaf>,
10768    group_id: Option<String>,
10769}
10770
10771impl SelectCall {
10772    fn new(ctx: WorkflowContext, operations: Vec<(SelectionKey, ParallelOperation)>) -> Self {
10773        Self {
10774            ctx,
10775            operations: Some(operations),
10776            members: Vec::new(),
10777            leaves: Vec::new(),
10778            group_id: None,
10779        }
10780    }
10781
10782    fn initialize(&mut self) -> Result<()> {
10783        let operations = self.operations.take().unwrap_or_default();
10784        let base_sequence = {
10785            let state = self
10786                .ctx
10787                .state
10788                .lock()
10789                .map_err(|_| Error::WorkflowStatePoisoned)?;
10790            if let Some(marker) = state.selection_markers.get(state.selection_marker_cursor) {
10791                marker.selection_group_base_sequence
10792            } else if let Some(recorded) = state.recorded_commands.get(state.command_cursor) {
10793                recorded.sequence()
10794            } else {
10795                let last = state
10796                    .recorded_commands
10797                    .last()
10798                    .map(RecordedCommand::sequence)
10799                    .unwrap_or(0);
10800                last.checked_add(u64::try_from(state.commands.len()).unwrap_or(u64::MAX))
10801                    .and_then(|sequence| sequence.checked_add(1))
10802                    .ok_or(Error::TimerDurationOverflow)?
10803            }
10804        };
10805        let (descriptors, members) = selection_descriptors(operations, base_sequence)?;
10806        let group_id = format!("select-calls:{base_sequence}:{}", descriptors.len());
10807        self.leaves = descriptors
10808            .into_iter()
10809            .map(|descriptor| SelectionLeaf {
10810                call: parallel_leaf_call(&self.ctx, descriptor.operation, descriptor.group_path),
10811                outcome: None,
10812            })
10813            .collect();
10814        self.members = members;
10815        self.group_id = Some(group_id);
10816        Ok(())
10817    }
10818}
10819
10820impl Future for SelectCall {
10821    type Output = Result<SelectionResult>;
10822
10823    fn poll(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
10824        if self.operations.is_some() {
10825            if let Err(error) = self.initialize() {
10826                return Poll::Ready(Err(error));
10827            }
10828        }
10829
10830        for leaf in &mut self.leaves {
10831            if leaf.outcome.is_some() {
10832                continue;
10833            }
10834            if let Poll::Ready(outcome) = leaf.call.poll_avro_value(cx) {
10835                if outcome
10836                    .as_ref()
10837                    .err()
10838                    .is_some_and(workflow_task_integrity_error)
10839                {
10840                    return Poll::Ready(outcome.map(|_| unreachable!()));
10841                }
10842                leaf.outcome = Some(outcome);
10843            }
10844        }
10845
10846        let all_members_terminal = self.leaves.iter().all(|leaf| leaf.outcome.is_some());
10847        let selection_member_range = self
10848            .members
10849            .first()
10850            .map(|member| member.base_sequence)
10851            .zip(self.leaves.len().try_into().ok())
10852            .map(|(base_sequence, size): (u64, u64)| {
10853                base_sequence..base_sequence.saturating_add(size)
10854            });
10855        let marker = {
10856            let mut state = match self.ctx.state.lock() {
10857                Ok(state) => state,
10858                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10859            };
10860            let marker = state
10861                .selection_markers
10862                .get(state.selection_marker_cursor)
10863                .cloned();
10864            if marker.is_none()
10865                && all_members_terminal
10866                && selection_member_range.as_ref().is_some_and(|member_range| {
10867                    state
10868                        .recorded_commands
10869                        .iter()
10870                        .any(|command| member_range.contains(&command.sequence()))
10871                })
10872            {
10873                // Terminal member history can be committed before the server's
10874                // SelectionResolved marker becomes visible to this task. That
10875                // marker is the durable barrier the selection is still waiting
10876                // on, so an otherwise commandless replay remains legitimately
10877                // pending instead of failing as an untracked yield.
10878                state.matched_recorded_pending = true;
10879            }
10880            marker
10881        };
10882        let Some(marker) = marker else {
10883            return Poll::Pending;
10884        };
10885        if self.group_id.as_deref() != Some(marker.selection_group_id.as_str())
10886            || marker.selection_group_size != self.leaves.len()
10887            || self.members.first().map(|member| member.base_sequence)
10888                != Some(marker.selection_group_base_sequence)
10889        {
10890            return Poll::Ready(Err(invalid_recorded_history(
10891                "selection_group_shape_mismatch",
10892                marker.selection_group_base_sequence,
10893                self.group_id
10894                    .as_deref()
10895                    .unwrap_or("initialized selection group"),
10896                &marker.selection_group_id,
10897                "recorded selection group differs from current workflow code",
10898            )));
10899        }
10900        let Some(member_position) = self.members.iter().position(|member| {
10901            member.key == marker.member_key
10902                && member.index == marker.member_index
10903                && member.base_sequence == marker.member_base_sequence
10904                && member.size == marker.member_size
10905                && member.kind == marker.operation_kind
10906        }) else {
10907            return Poll::Ready(Err(invalid_recorded_history(
10908                "selection_member_shape_mismatch",
10909                marker.member_base_sequence,
10910                "winner member matching current workflow code",
10911                &format!("{:?}", marker.member_key),
10912                "recorded selection winner differs from the authored member identity",
10913            )));
10914        };
10915        let member = self.members[member_position].clone();
10916        let (handles, resolution_sequence) = {
10917            let mut state = match self.ctx.state.lock() {
10918                Ok(state) => state,
10919                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
10920            };
10921            let identities = self
10922                .members
10923                .iter()
10924                .map(|candidate| {
10925                    selection_operation_identity(
10926                        &state,
10927                        &candidate.kind,
10928                        candidate.base_sequence,
10929                        candidate.size,
10930                    )
10931                })
10932                .collect::<Vec<_>>();
10933            if let Some((position, missing)) = identities
10934                .iter()
10935                .enumerate()
10936                .find(|(_, identity)| identity.is_empty())
10937                .map(|(position, identity)| (position, identity.clone()))
10938            {
10939                let candidate = &self.members[position];
10940                return Poll::Ready(Err(invalid_recorded_history(
10941                    "selection_operation_identity_missing",
10942                    candidate.base_sequence,
10943                    &format!(
10944                        "durable {} resource identity from scheduled/open history",
10945                        candidate.kind
10946                    ),
10947                    &missing,
10948                    "selection member history is missing its canonical durable identity",
10949                )));
10950            }
10951            let expected_winner_identity = &identities[member_position];
10952            let resolution_sequence = match validated_selection_resolution_sequence(
10953                &state,
10954                &marker,
10955                &member,
10956                expected_winner_identity,
10957            ) {
10958                Ok(sequence) => sequence,
10959                Err(error) => return Poll::Ready(Err(error)),
10960            };
10961            let handles = self
10962                .members
10963                .iter()
10964                .zip(identities)
10965                .map(|(member, identity)| DurableOperationHandle {
10966                    ctx: self.ctx.clone(),
10967                    key: member.key.clone(),
10968                    index: member.index,
10969                    kind: member.kind.clone(),
10970                    identity,
10971                    base_sequence: member.base_sequence,
10972                    size: member.size,
10973                    selection_group_id: marker.selection_group_id.clone(),
10974                    shape: member.shape.clone(),
10975                })
10976                .collect::<Vec<_>>();
10977            if let Err(error) = validate_selection_cancellations_for_handles(&state, &handles) {
10978                return Poll::Ready(Err(error));
10979            }
10980            state.selection_marker_cursor += 1;
10981            (handles, resolution_sequence)
10982        };
10983
10984        let mut winner_failure = None;
10985        let mut flat_results = Vec::with_capacity(member.size);
10986        if marker.outcome == "failed" {
10987            let resolution_offset = match resolution_sequence
10988                .checked_sub(member.base_sequence)
10989                .and_then(|offset| usize::try_from(offset).ok())
10990            {
10991                Some(offset) if offset < member.size => offset,
10992                _ => {
10993                    return Poll::Ready(Err(invalid_recorded_history(
10994                        "selection_resolution_event_mismatch",
10995                        member.base_sequence,
10996                        "failure event within selected member bounds",
10997                        &resolution_sequence.to_string(),
10998                        "selection failure event is outside the authored member",
10999                    )))
11000                }
11001            };
11002            let leaf = &mut self.leaves[member.leaf_start + resolution_offset];
11003            match leaf.outcome.take() {
11004                Some(Err(error)) => winner_failure = Some(error),
11005                _ => {
11006                    return Poll::Ready(Err(invalid_recorded_history(
11007                        "selection_winner_outcome_mismatch",
11008                        member.base_sequence,
11009                        "exact failed terminal history referenced by SelectionResolved",
11010                        "missing or successful resolution event",
11011                        "selection winner marker disagrees with terminal operation history",
11012                    )))
11013                }
11014            }
11015        } else {
11016            for leaf in &mut self.leaves[member.leaf_start..member.leaf_start + member.size] {
11017                match leaf.outcome.take() {
11018                    Some(Ok(result)) => flat_results.push(result),
11019                    Some(Err(_)) => {
11020                        return Poll::Ready(Err(invalid_recorded_history(
11021                            "selection_winner_outcome_mismatch",
11022                            member.base_sequence,
11023                            "fully completed nested selection member",
11024                            "failed durable leaf",
11025                            "completed selection winner contains a failed leaf",
11026                        )))
11027                    }
11028                    None => {
11029                        return Poll::Ready(Err(invalid_recorded_history(
11030                            "selection_winner_unresolved",
11031                            member.base_sequence,
11032                            "terminal history for every completed winner leaf",
11033                            "pending member history",
11034                            "completed SelectionResolved member has an unfinished durable barrier",
11035                        )))
11036                    }
11037                }
11038            }
11039        }
11040        let value = if winner_failure.is_none() {
11041            let mut flat_results = flat_results.into_iter();
11042            let value = parallel_results_for_shape(&member.shape, &mut flat_results);
11043            match value.into_json_result() {
11044                Ok(value) => Some(value),
11045                Err(error) => return Poll::Ready(Err(error)),
11046            }
11047        } else {
11048            None
11049        };
11050        let winner = handles[member_position].clone();
11051        Poll::Ready(Ok(SelectionResult {
11052            key: winner.key.clone(),
11053            index: winner.index,
11054            kind: winner.kind.clone(),
11055            identity: winner.identity.clone(),
11056            value,
11057            failure: winner_failure,
11058            winner,
11059            handles,
11060        }))
11061    }
11062}
11063
11064fn selection_operation_identity(
11065    state: &WorkflowState,
11066    kind: &str,
11067    base_sequence: u64,
11068    size: usize,
11069) -> String {
11070    if kind == "group" {
11071        return format!("group:{base_sequence}:{size}");
11072    }
11073    let fields: &[&str] = match kind {
11074        "activity" => &["activity_execution_id"],
11075        "child" => &["child_workflow_run_id"],
11076        "timer" => &["timer_id"],
11077        "signal" => &["signal_wait_id"],
11078        "condition" => &["condition_wait_id"],
11079        _ => &[],
11080    };
11081    for sequence in base_sequence..base_sequence.saturating_add(size as u64) {
11082        for event in state
11083            .history_events
11084            .iter()
11085            .filter(|event| durable_event_sequence(event) == Some(sequence))
11086        {
11087            for field in fields {
11088                if let Some(identity) = event.payload.get(*field).and_then(Value::as_str) {
11089                    if !identity.is_empty() {
11090                        return identity.to_string();
11091                    }
11092                }
11093            }
11094        }
11095    }
11096    String::new()
11097}
11098
11099fn validated_selection_resolution_sequence(
11100    state: &WorkflowState,
11101    marker: &SelectionMarker,
11102    member: &SelectionMemberPlan,
11103    expected_identity: &str,
11104) -> Result<u64> {
11105    if expected_identity.is_empty() {
11106        return Err(invalid_recorded_history(
11107            "selection_operation_identity_missing",
11108            member.base_sequence,
11109            &format!(
11110                "durable {} resource identity from scheduled/open history",
11111                member.kind
11112            ),
11113            "missing operation identity",
11114            "selection member history is missing its canonical durable identity",
11115        ));
11116    }
11117    if marker.operation_identity != expected_identity {
11118        return Err(invalid_recorded_history(
11119            "selection_operation_identity_mismatch",
11120            member.base_sequence,
11121            expected_identity,
11122            &marker.operation_identity,
11123            "selection winner identity does not match durable scheduled/open history",
11124        ));
11125    }
11126
11127    let failure_types = [
11128        "ActivityFailed",
11129        "ActivityCancelled",
11130        "ActivityTimedOut",
11131        "ChildRunFailed",
11132        "ChildRunCancelled",
11133        "ChildRunTerminated",
11134    ];
11135    let success_types = [
11136        "ActivityCompleted",
11137        "ChildRunCompleted",
11138        "TimerFired",
11139        "SignalApplied",
11140        "ConditionWaitSatisfied",
11141        "ConditionWaitTimedOut",
11142    ];
11143    let terminal_types: &[&str] = if marker.outcome == "failed" {
11144        &failure_types
11145    } else {
11146        &success_types
11147    };
11148    let mut candidates = Vec::new();
11149    for event in state.history_events.iter() {
11150        let Some(sequence) = durable_event_sequence(event) else {
11151            continue;
11152        };
11153        if sequence < member.base_sequence
11154            || sequence >= member.base_sequence.saturating_add(member.size as u64)
11155            || !terminal_types.contains(&event.event_type.as_str())
11156        {
11157            continue;
11158        }
11159        let event_id = event
11160            .raw
11161            .get("id")
11162            .or_else(|| event.raw.get("event_id"))
11163            .and_then(Value::as_str)
11164            .filter(|value| !value.is_empty())
11165            .ok_or_else(|| {
11166                invalid_recorded_history(
11167                    "selection_resolution_event_id_missing",
11168                    member.base_sequence,
11169                    "terminal selection history with a durable event id",
11170                    &event.payload.to_string(),
11171                    "selection terminal history cannot be bound to its winner marker",
11172                )
11173            })?;
11174        candidates.push((event_id.to_string(), event.event_type.clone(), sequence));
11175    }
11176    let resolution = if marker.outcome == "failed" {
11177        candidates.first()
11178    } else {
11179        candidates.last()
11180    };
11181    let Some((event_id, event_type, sequence)) = resolution else {
11182        return Err(invalid_recorded_history(
11183            "selection_resolution_event_missing",
11184            member.base_sequence,
11185            "terminal history for the selected member",
11186            &format!("{:?}", marker.member_key),
11187            "selection winner marker has no matching durable terminal event",
11188        ));
11189    };
11190    if event_id != &marker.resolution_event_id || event_type != &marker.resolution_event_type {
11191        return Err(invalid_recorded_history(
11192            "selection_resolution_event_mismatch",
11193            member.base_sequence,
11194            &format!("{event_type}:{event_id}"),
11195            &format!(
11196                "{}:{}",
11197                marker.resolution_event_type, marker.resolution_event_id
11198            ),
11199            "selection winner marker does not reference the event that made its member terminal",
11200        ));
11201    }
11202    Ok(*sequence)
11203}
11204
11205fn recorded_selection_member_outcome(
11206    state: &WorkflowState,
11207    handle: &DurableOperationHandle,
11208) -> Result<Option<ParallelResult>> {
11209    for event in state.history_events.iter() {
11210        let Some(sequence) = durable_event_sequence(event) else {
11211            continue;
11212        };
11213        if sequence < handle.base_sequence
11214            || sequence >= handle.base_sequence.saturating_add(handle.size as u64)
11215            || !matches!(
11216                event.event_type.as_str(),
11217                "ActivityFailed"
11218                    | "ActivityCancelled"
11219                    | "ActivityTimedOut"
11220                    | "ChildRunFailed"
11221                    | "ChildRunCancelled"
11222                    | "ChildRunTerminated"
11223            )
11224        {
11225            continue;
11226        }
11227        let Some(command) = state
11228            .recorded_commands
11229            .iter()
11230            .find(|command| command.sequence() == sequence)
11231        else {
11232            continue;
11233        };
11234        match command {
11235            RecordedCommand::Activity {
11236                outcome: Some(Err(failure)),
11237                ..
11238            } => return Err(Error::ActivityFailed(failure.clone())),
11239            RecordedCommand::ChildWorkflow {
11240                outcome: Some(Err(failure)),
11241                ..
11242            } => return Err(Error::ChildWorkflowFailed(failure.clone())),
11243            _ => {}
11244        }
11245    }
11246
11247    let mut results = Vec::with_capacity(handle.size);
11248    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11249        let Some(command) = state
11250            .recorded_commands
11251            .iter()
11252            .find(|command| command.sequence() == sequence)
11253        else {
11254            return Ok(None);
11255        };
11256        let result = match command {
11257            RecordedCommand::Activity { outcome, .. } => match outcome {
11258                Some(Ok(value)) => ParallelAvroResult::Activity(value.clone()),
11259                Some(Err(failure)) => return Err(Error::ActivityFailed(failure.clone())),
11260                None => return Ok(None),
11261            },
11262            RecordedCommand::Timer { fired, .. } => {
11263                if !fired {
11264                    return Ok(None);
11265                }
11266                ParallelAvroResult::Timer
11267            }
11268            RecordedCommand::ChildWorkflow { outcome, .. } => match outcome {
11269                Some(Ok(value)) => ParallelAvroResult::ChildWorkflow(value.clone()),
11270                Some(Err(failure)) => return Err(Error::ChildWorkflowFailed(failure.clone())),
11271                None => return Ok(None),
11272            },
11273            RecordedCommand::SignalWait { value, .. } => match value {
11274                Some(value) => ParallelAvroResult::Signal(value.clone()),
11275                None => return Ok(None),
11276            },
11277            RecordedCommand::ConditionWait { result, .. } => match result {
11278                Some(result) => ParallelAvroResult::Condition(*result),
11279                None => return Ok(None),
11280            },
11281            other => {
11282                return Err(command_mismatch(
11283                    other,
11284                    format!("selected {} member", handle.kind),
11285                ))
11286            }
11287        };
11288        results.push(result);
11289    }
11290    let mut results = results.into_iter();
11291    parallel_results_for_shape(&handle.shape, &mut results)
11292        .into_json_result()
11293        .map(Some)
11294}
11295
11296fn recorded_selection_member_is_terminal(
11297    state: &WorkflowState,
11298    handle: &DurableOperationHandle,
11299) -> bool {
11300    let mut completed = 0usize;
11301    let mut all_completed = true;
11302    for sequence in handle.base_sequence..handle.base_sequence.saturating_add(handle.size as u64) {
11303        let Some(command) = state
11304            .recorded_commands
11305            .iter()
11306            .find(|command| command.sequence() == sequence)
11307        else {
11308            all_completed = false;
11309            continue;
11310        };
11311        let terminal = match command {
11312            RecordedCommand::Activity {
11313                outcome: Some(Err(_)),
11314                ..
11315            }
11316            | RecordedCommand::ChildWorkflow {
11317                outcome: Some(Err(_)),
11318                ..
11319            } => return true,
11320            RecordedCommand::Activity { outcome, .. } => outcome.is_some(),
11321            RecordedCommand::ChildWorkflow { outcome, .. } => outcome.is_some(),
11322            RecordedCommand::Timer { fired, .. } => *fired,
11323            RecordedCommand::SignalWait { value, .. } => value.is_some(),
11324            RecordedCommand::ConditionWait { result, .. } => result.is_some(),
11325            RecordedCommand::SearchAttributes { .. }
11326            | RecordedCommand::SideEffect { .. }
11327            | RecordedCommand::VersionMarker { .. }
11328            | RecordedCommand::Memo { .. } => false,
11329        };
11330        if !terminal {
11331            all_completed = false;
11332            continue;
11333        }
11334        completed += 1;
11335    }
11336    all_completed && completed == handle.size
11337}
11338
11339fn selection_cancellation_for_handle(
11340    state: &WorkflowState,
11341    handle: &DurableOperationHandle,
11342) -> Result<bool> {
11343    let Some(marker) = state.cancelled_selection_members.iter().find(|recorded| {
11344        recorded.selection_group_id == handle.selection_group_id
11345            && recorded.member_base_sequence == handle.base_sequence
11346    }) else {
11347        return Ok(false);
11348    };
11349    validate_selection_cancellation_marker(marker, handle)?;
11350    Ok(true)
11351}
11352
11353fn validate_selection_cancellations_for_handles(
11354    state: &WorkflowState,
11355    handles: &[DurableOperationHandle],
11356) -> Result<()> {
11357    let Some(group_id) = handles
11358        .first()
11359        .map(|handle| handle.selection_group_id.as_str())
11360    else {
11361        return Ok(());
11362    };
11363    for marker in state
11364        .cancelled_selection_members
11365        .iter()
11366        .filter(|marker| marker.selection_group_id == group_id)
11367    {
11368        let Some(handle) = handles
11369            .iter()
11370            .find(|handle| handle.base_sequence == marker.member_base_sequence)
11371        else {
11372            return Err(invalid_recorded_history(
11373                "selection_cancellation_member_mismatch",
11374                marker.member_base_sequence,
11375                "SelectionOperationCancelled matching an authored selection handle",
11376                &format!("{marker:?}"),
11377                "selection cancellation member base does not name an authored member",
11378            ));
11379        };
11380        validate_selection_cancellation_marker(marker, handle)?;
11381    }
11382    Ok(())
11383}
11384
11385fn validate_selection_cancellation_marker(
11386    marker: &SelectionCancellation,
11387    handle: &DurableOperationHandle,
11388) -> Result<()> {
11389    if marker.selection_group_id != handle.selection_group_id
11390        || marker.member_key != handle.key
11391        || marker.member_index != handle.index
11392        || marker.member_base_sequence != handle.base_sequence
11393        || marker.member_size != handle.size
11394        || marker.operation_kind != handle.kind
11395        || marker.operation_identity != handle.identity
11396    {
11397        return Err(invalid_recorded_history(
11398            "selection_cancellation_member_mismatch",
11399            handle.base_sequence,
11400            "SelectionOperationCancelled matching the authored selection handle",
11401            &format!("{marker:?}"),
11402            "selection cancellation history targets different authored member metadata",
11403        ));
11404    }
11405    Ok(())
11406}
11407
11408/// Future returned by [`DurableOperationHandle::await_result`].
11409pub struct DurableOperationAwaitCall {
11410    handle: DurableOperationHandle,
11411}
11412
11413impl Future for DurableOperationAwaitCall {
11414    type Output = Result<ParallelResult>;
11415
11416    fn poll(self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11417        let state = match self.handle.ctx.state.lock() {
11418            Ok(state) => state,
11419            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11420        };
11421        match selection_cancellation_for_handle(&state, &self.handle) {
11422            Err(error) => return Poll::Ready(Err(error)),
11423            Ok(false) => {}
11424            Ok(true) => {
11425                return Poll::Ready(Err(Error::DurableOperationCancelled(
11426                    DurableOperationCancelled {
11427                        selection_group_id: self.handle.selection_group_id.clone(),
11428                        member_key: self.handle.key.clone(),
11429                        member_index: self.handle.index,
11430                        operation_kind: self.handle.kind.clone(),
11431                        operation_identity: self.handle.identity.clone(),
11432                    },
11433                )));
11434            }
11435        }
11436        match recorded_selection_member_outcome(&state, &self.handle) {
11437            Ok(Some(result)) => Poll::Ready(Ok(result)),
11438            Ok(None) => Poll::Pending,
11439            Err(error) => Poll::Ready(Err(error)),
11440        }
11441    }
11442}
11443
11444/// Future returned by [`DurableOperationHandle::cancel`].
11445pub struct CancelDurableOperationCall {
11446    handle: DurableOperationHandle,
11447    emitted: bool,
11448}
11449
11450impl Future for CancelDurableOperationCall {
11451    type Output = Result<()>;
11452
11453    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11454        let ctx = self.handle.ctx.clone();
11455        let mut state = match ctx.state.lock() {
11456            Ok(state) => state,
11457            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11458        };
11459        match selection_cancellation_for_handle(&state, &self.handle) {
11460            Err(error) => return Poll::Ready(Err(error)),
11461            Ok(true) => return Poll::Ready(Ok(())),
11462            Ok(false) => {}
11463        }
11464        if recorded_selection_member_is_terminal(&state, &self.handle) {
11465            return Poll::Ready(Ok(()));
11466        }
11467        if !self.emitted {
11468            state.commands.push(json!({
11469                "type": "cancel_selection_operation",
11470                "selection_group_id": self.handle.selection_group_id,
11471                "member_key": self.handle.key,
11472                "member_index": self.handle.index,
11473                "member_base_sequence": self.handle.base_sequence,
11474                "member_size": self.handle.size,
11475                "operation_kind": self.handle.kind,
11476                "operation_identity": self.handle.identity,
11477            }));
11478            self.emitted = true;
11479        }
11480        // The cancellation command is only a request. Do not expose workflow
11481        // state after cancel until committed SelectionOperationCancelled
11482        // history proves that the request won the race with member completion.
11483        Poll::Pending
11484    }
11485}
11486
11487trait PollNestedResultExt<T> {
11488    fn flatten_result(self) -> Poll<Result<T>>;
11489}
11490
11491impl<T> PollNestedResultExt<T> for Poll<Result<Result<T>>> {
11492    fn flatten_result(self) -> Poll<Result<T>> {
11493        match self {
11494            Poll::Ready(Ok(result)) => Poll::Ready(result),
11495            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11496            Poll::Pending => Poll::Pending,
11497        }
11498    }
11499}
11500
11501struct SagaCompensation {
11502    activity_type: String,
11503    options: ActivityOptions,
11504    arguments: AvroValue,
11505    registration_order: usize,
11506}
11507
11508/// Workflow-local deterministic saga compensation helper.
11509///
11510/// Register each compensation only after its forward step succeeds. Passing
11511/// the forward `Result` to [`Saga::finish`] runs compensations sequentially in
11512/// reverse registration order after any failure, including cooperative
11513/// cancellation. Each compensation is an ordinary durable activity, so replay,
11514/// duplicate delivery, and worker restart use existing history semantics.
11515pub struct Saga {
11516    ctx: WorkflowContext,
11517    compensations: Vec<SagaCompensation>,
11518}
11519
11520impl Saga {
11521    fn new(ctx: WorkflowContext) -> Self {
11522        Self {
11523            ctx,
11524            compensations: Vec::new(),
11525        }
11526    }
11527
11528    pub fn add_compensation<T: Serialize>(
11529        &mut self,
11530        activity_type: impl Into<String>,
11531        args: T,
11532    ) -> Result<&mut Self> {
11533        self.add_compensation_with_options(activity_type, ActivityOptions::new(), args)
11534    }
11535
11536    pub fn add_compensation_with_options<T: Serialize>(
11537        &mut self,
11538        activity_type: impl Into<String>,
11539        options: ActivityOptions,
11540        args: T,
11541    ) -> Result<&mut Self> {
11542        let activity_type = activity_type.into();
11543        if activity_type.trim().is_empty() || activity_type.trim() != activity_type {
11544            return Err(Error::Codec(
11545                "saga compensation activity type must be non-empty without surrounding whitespace"
11546                    .to_string(),
11547            ));
11548        }
11549        options.validate().map_err(Error::InvalidActivityOptions)?;
11550        let arguments = AvroValue::from_serialize(&args)?;
11551        let registration_order = self.compensations.len() + 1;
11552        self.compensations.push(SagaCompensation {
11553            activity_type,
11554            options,
11555            arguments,
11556            registration_order,
11557        });
11558        Ok(self)
11559    }
11560
11561    /// Compensate `initiating_failure` and return the failure that must remain.
11562    pub async fn compensate(mut self, initiating_failure: Error) -> Error {
11563        while let Some(compensation) = self.compensations.pop() {
11564            if let Err(compensation_failure) = self
11565                .ctx
11566                .activity_with_options(
11567                    compensation.activity_type.clone(),
11568                    compensation.options,
11569                    compensation.arguments,
11570                )
11571                .await
11572            {
11573                if workflow_task_integrity_error(&compensation_failure) {
11574                    return compensation_failure;
11575                }
11576                return Error::SagaCompensationFailed(SagaCompensationFailure {
11577                    initiating_failure: Box::new(initiating_failure),
11578                    compensation_failure: Box::new(compensation_failure),
11579                    compensation_activity_type: compensation.activity_type,
11580                    compensation_registration_order: compensation.registration_order,
11581                });
11582            }
11583        }
11584        initiating_failure
11585    }
11586
11587    /// Return a successful forward value or compensate and preserve its failure.
11588    pub async fn finish<T>(self, outcome: Result<T>) -> Result<T> {
11589        match outcome {
11590            Ok(value) => Ok(value),
11591            Err(error) => Err(self.compensate(error).await),
11592        }
11593    }
11594}
11595
11596pub struct ActivityCall {
11597    ctx: WorkflowContext,
11598    activity_type: String,
11599    options: ActivityOptions,
11600    args: Option<Result<AvroValue>>,
11601    scheduled: bool,
11602    parallel_group_path: Vec<ParallelGroupMetadata>,
11603}
11604
11605impl ActivityCall {
11606    fn poll_avro_value(
11607        mut self: Pin<&mut Self>,
11608        _cx: &mut TaskContext<'_>,
11609    ) -> Poll<Result<AvroValue>> {
11610        let ctx = self.ctx.clone();
11611        let mut state = match ctx.state.lock() {
11612            Ok(state) => state,
11613            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11614        };
11615
11616        if self.scheduled {
11617            return Poll::Pending;
11618        }
11619
11620        let options = match self.options.validate() {
11621            Ok(options) => options,
11622            Err(error) => {
11623                return Poll::Ready(Err(Error::InvalidActivityOptions(error)));
11624            }
11625        };
11626        let task_queue = options
11627            .task_queue
11628            .clone()
11629            .unwrap_or_else(|| state.task_queue.clone());
11630        let current_recorded_options = RecordedActivityOptions {
11631            task_queue: RecordedSnapshotValue::Known(Some(task_queue.clone())),
11632            // Rust schedules ordinary durable activities. The server records a
11633            // non-null mode only for a specialized execution primitive.
11634            execution_mode: RecordedSnapshotValue::Known(None),
11635            retry_policy: current_activity_retry_snapshot(&options),
11636        };
11637
11638        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11639            let sequence = recorded.sequence();
11640            match recorded {
11641                RecordedCommand::Activity {
11642                    activity_type,
11643                    options: recorded_options,
11644                    outcome,
11645                    parallel_group_path,
11646                    ..
11647                } => {
11648                    if let Err(error) = ensure_parallel_path_matches(
11649                        sequence,
11650                        parallel_group_path.as_deref(),
11651                        &self.parallel_group_path,
11652                    ) {
11653                        return Poll::Ready(Err(error));
11654                    }
11655                    if let Some(recorded_type) = activity_type {
11656                        if recorded_type != self.activity_type {
11657                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11658                                ReplayFailure::new(
11659                                    "recorded_command_detail_mismatch",
11660                                    Some(sequence),
11661                                    Some(format!("activity:{recorded_type}")),
11662                                    Some(format!("activity:{}", self.activity_type)),
11663                                    "recorded activity type differs from the current workflow command",
11664                                ),
11665                            )));
11666                        }
11667                    }
11668                    if let Some(recorded_options) = recorded_options {
11669                        if !recorded_options
11670                            .task_queue
11671                            .matches_current(&current_recorded_options.task_queue)
11672                        {
11673                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11674                                ReplayFailure::new(
11675                                    "activity_task_queue_mismatch",
11676                                    Some(sequence),
11677                                    Some(activity_options_description(&recorded_options)),
11678                                    Some(activity_options_description(&current_recorded_options)),
11679                                    "recorded activity task queue differs from the current workflow command",
11680                                ),
11681                            )));
11682                        }
11683                        if !recorded_options
11684                            .execution_mode
11685                            .matches_current(&current_recorded_options.execution_mode)
11686                        {
11687                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11688                                ReplayFailure::new(
11689                                    "activity_execution_mode_mismatch",
11690                                    Some(sequence),
11691                                    Some(activity_options_description(&recorded_options)),
11692                                    Some(activity_options_description(&current_recorded_options)),
11693                                    "recorded activity execution mode differs from the current workflow command",
11694                                ),
11695                            )));
11696                        }
11697                        if !recorded_options
11698                            .retry_policy
11699                            .matches_current(&current_recorded_options.retry_policy)
11700                        {
11701                            return Poll::Ready(Err(Error::NonDeterministicReplay(
11702                                ReplayFailure::new(
11703                                    "activity_retry_policy_mismatch",
11704                                    Some(sequence),
11705                                    Some(activity_options_description(&recorded_options)),
11706                                    Some(activity_options_description(&current_recorded_options)),
11707                                    "recorded activity retry policy differs from the current workflow command",
11708                                ),
11709                            )));
11710                        }
11711                    }
11712                    state.command_cursor += 1;
11713                    if let Some(outcome) = outcome {
11714                        return Poll::Ready(outcome.map_err(Error::ActivityFailed));
11715                    }
11716                    state.matched_recorded_pending = true;
11717                    self.scheduled = true;
11718                    return Poll::Pending;
11719                }
11720                other => {
11721                    return Poll::Ready(Err(command_mismatch(
11722                        &other,
11723                        format!("activity:{}", self.activity_type),
11724                    )));
11725                }
11726            }
11727        }
11728
11729        if !self.scheduled {
11730            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
11731                Ok(args) => args,
11732                Err(error) => return Poll::Ready(Err(error)),
11733            };
11734            let arguments = normalize_avro_arguments(args);
11735            let envelope = match encode_typed_envelope(&arguments, &state.payload_codec) {
11736                Ok(envelope) => envelope,
11737                Err(error) => return Poll::Ready(Err(error)),
11738            };
11739
11740            let mut command = serde_json::Map::from_iter([
11741                ("type".to_string(), json!("schedule_activity")),
11742                (
11743                    "activity_type".to_string(),
11744                    json!(self.activity_type.clone()),
11745                ),
11746                ("queue".to_string(), json!(task_queue)),
11747                ("arguments".to_string(), envelope),
11748            ]);
11749            for (field, value) in [
11750                ("start_to_close_timeout", options.start_to_close_timeout),
11751                (
11752                    "schedule_to_start_timeout",
11753                    options.schedule_to_start_timeout,
11754                ),
11755                (
11756                    "schedule_to_close_timeout",
11757                    options.schedule_to_close_timeout,
11758                ),
11759                ("heartbeat_timeout", options.heartbeat_timeout),
11760            ] {
11761                if let Some(value) = value {
11762                    command.insert(field.to_string(), json!(value));
11763                }
11764            }
11765            if let Some(retry_policy) = options.retry_policy {
11766                command.insert("retry_policy".to_string(), retry_policy);
11767            }
11768            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11769            state.commands.push(Value::Object(command));
11770            self.scheduled = true;
11771        }
11772
11773        Poll::Pending
11774    }
11775}
11776
11777impl Future for ActivityCall {
11778    type Output = Result<Value>;
11779
11780    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11781        match self.poll_avro_value(cx) {
11782            Poll::Ready(Ok(value)) => Poll::Ready(value.into_json()),
11783            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
11784            Poll::Pending => Poll::Pending,
11785        }
11786    }
11787}
11788
11789/// Future returned by [`WorkflowContext::sleep`].
11790pub struct TimerCall {
11791    ctx: WorkflowContext,
11792    delay_seconds: Option<u64>,
11793    scheduled: bool,
11794    matched_pending: bool,
11795    parallel_group_path: Vec<ParallelGroupMetadata>,
11796}
11797
11798impl Future for TimerCall {
11799    type Output = Result<()>;
11800
11801    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11802        if self.matched_pending {
11803            return Poll::Pending;
11804        }
11805
11806        let ctx = self.ctx.clone();
11807        let Some(requested_delay) = self.delay_seconds else {
11808            return Poll::Ready(Err(Error::TimerDurationOverflow));
11809        };
11810        let mut state = match ctx.state.lock() {
11811            Ok(state) => state,
11812            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11813        };
11814
11815        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
11816            match recorded {
11817                RecordedCommand::Timer {
11818                    sequence,
11819                    delay_seconds,
11820                    fired,
11821                    parallel_group_path,
11822                    ..
11823                } => {
11824                    if let Err(error) = ensure_parallel_path_matches(
11825                        sequence,
11826                        parallel_group_path.as_deref(),
11827                        &self.parallel_group_path,
11828                    ) {
11829                        return Poll::Ready(Err(error));
11830                    }
11831                    if delay_seconds != requested_delay {
11832                        return Poll::Ready(Err(Error::NonDeterministicReplay(
11833                            ReplayFailure::new(
11834                                "timer_delay_mismatch",
11835                                Some(sequence),
11836                                Some(format!("timer:{delay_seconds}s")),
11837                                Some(format!("timer:{requested_delay}s")),
11838                                "recorded timer delay differs from the current workflow command",
11839                            ),
11840                        )));
11841                    }
11842                    state.command_cursor += 1;
11843                    if fired {
11844                        return Poll::Ready(Ok(()));
11845                    }
11846                    state.matched_recorded_pending = true;
11847                    self.scheduled = true;
11848                    self.matched_pending = true;
11849                    return Poll::Pending;
11850                }
11851                other => return Poll::Ready(Err(command_mismatch(&other, "timer"))),
11852            }
11853        }
11854
11855        if !self.scheduled {
11856            let mut command = serde_json::Map::from_iter([
11857                ("type".to_string(), json!("start_timer")),
11858                ("delay_seconds".to_string(), json!(requested_delay)),
11859            ]);
11860            apply_parallel_group_path(&mut command, &self.parallel_group_path);
11861            state.commands.push(Value::Object(command));
11862            self.scheduled = true;
11863        }
11864
11865        Poll::Pending
11866    }
11867}
11868
11869/// Future returned by [`WorkflowContext::wait_condition`].
11870pub struct ConditionWaitCall {
11871    ctx: WorkflowContext,
11872    options: ConditionWaitOptions,
11873    predicate: Box<dyn Fn() -> Result<bool> + Send + 'static>,
11874    occurrence_id: Option<String>,
11875    opened_wait: bool,
11876    parallel_group_path: Vec<ParallelGroupMetadata>,
11877}
11878
11879impl Future for ConditionWaitCall {
11880    type Output = Result<ConditionWaitResult>;
11881
11882    fn poll(mut self: Pin<&mut Self>, _cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
11883        if self.opened_wait {
11884            return Poll::Pending;
11885        }
11886
11887        let options = match self.options.validate() {
11888            Ok(options) => options,
11889            Err(error) => return Poll::Ready(Err(Error::InvalidConditionWaitOptions(error))),
11890        };
11891        let ctx = self.ctx.clone();
11892        let occurrence_id = match self.occurrence_id.as_ref() {
11893            Some(occurrence_id) => occurrence_id.clone(),
11894            None => {
11895                let mut state = match ctx.state.lock() {
11896                    Ok(state) => state,
11897                    Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11898                };
11899                let ordinal = state.condition_wait_occurrence_counter;
11900                state.condition_wait_occurrence_counter = match ordinal.checked_add(1) {
11901                    Some(next) => next,
11902                    None => {
11903                        return Poll::Ready(Err(Error::WorkerLoop(
11904                            "condition wait occurrence counter overflowed".to_string(),
11905                        )))
11906                    }
11907                };
11908                let occurrence_id = format!("{CONDITION_WAIT_OCCURRENCE_PREFIX}{ordinal}");
11909                drop(state);
11910                self.occurrence_id = Some(occurrence_id.clone());
11911                occurrence_id
11912            }
11913        };
11914
11915        let recorded_result = {
11916            let mut state = match ctx.state.lock() {
11917                Ok(state) => state,
11918                Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
11919            };
11920            let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() else {
11921                drop(state);
11922                return self.poll_new_condition(options);
11923            };
11924            if !matches!(recorded, RecordedCommand::ConditionWait { .. }) {
11925                return Poll::Ready(Err(command_mismatch(&recorded, "condition wait")));
11926            }
11927
11928            let mut cursor = state.command_cursor;
11929            let mut result = None;
11930            loop {
11931                let Some(RecordedCommand::ConditionWait {
11932                    sequence,
11933                    occurrence_id: recorded_occurrence_id,
11934                    condition_key,
11935                    predicate_identity,
11936                    timeout_seconds,
11937                    result: recorded_result,
11938                    parallel_group_path,
11939                    ..
11940                }) = state.recorded_commands.get(cursor)
11941                else {
11942                    break;
11943                };
11944
11945                if cursor > state.command_cursor && recorded_occurrence_id != &occurrence_id {
11946                    break;
11947                }
11948                if let Err(error) = ensure_parallel_path_matches(
11949                    *sequence,
11950                    parallel_group_path.as_deref(),
11951                    &self.parallel_group_path,
11952                ) {
11953                    return Poll::Ready(Err(error));
11954                }
11955                if let Err(error) = validate_recorded_condition_wait(
11956                    *sequence,
11957                    recorded_occurrence_id,
11958                    condition_key.as_deref(),
11959                    predicate_identity,
11960                    *timeout_seconds,
11961                    &occurrence_id,
11962                    &options,
11963                ) {
11964                    return Poll::Ready(Err(error));
11965                }
11966                if result == Some(ConditionWaitResult::TimedOut) {
11967                    return Poll::Ready(Err(Error::NonDeterministicReplay(ReplayFailure::new(
11968                        "condition_wait_reopened_after_timeout",
11969                        Some(*sequence),
11970                        Some("timed-out condition is terminal".to_string()),
11971                        Some("another physical wait-open".to_string()),
11972                        "condition history reopened one logical wait after its durable timeout",
11973                    ))));
11974                }
11975                result = *recorded_result;
11976                cursor += 1;
11977            }
11978            state.command_cursor = cursor;
11979            result
11980        };
11981
11982        if let Some(result) = recorded_result {
11983            return Poll::Ready(Ok(result));
11984        }
11985
11986        self.poll_open_condition(options)
11987    }
11988}
11989
11990impl ConditionWaitCall {
11991    fn poll_new_condition(
11992        self: Pin<&mut Self>,
11993        options: ValidatedConditionWaitOptions,
11994    ) -> Poll<Result<ConditionWaitResult>> {
11995        self.poll_open_condition(options)
11996    }
11997
11998    fn poll_open_condition(
11999        mut self: Pin<&mut Self>,
12000        options: ValidatedConditionWaitOptions,
12001    ) -> Poll<Result<ConditionWaitResult>> {
12002        let selection_member = self
12003            .parallel_group_path
12004            .first()
12005            .is_some_and(|entry| entry.parallel_group_mode.as_deref() == Some("select"));
12006        match (self.predicate)() {
12007            Ok(true) if !selection_member => {
12008                return Poll::Ready(Ok(ConditionWaitResult::Satisfied))
12009            }
12010            Ok(_) => {}
12011            Err(error) => return Poll::Ready(Err(error)),
12012        }
12013        if options.timeout_seconds == Some(0) && !selection_member {
12014            return Poll::Ready(Ok(ConditionWaitResult::TimedOut));
12015        }
12016
12017        let ctx = self.ctx.clone();
12018        let mut state = match ctx.state.lock() {
12019            Ok(state) => state,
12020            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12021        };
12022        let mut command = serde_json::Map::from_iter([
12023            ("type".to_string(), json!("open_condition_wait")),
12024            (
12025                "condition_wait_occurrence_id".to_string(),
12026                json!(self.occurrence_id.as_deref().unwrap_or_default()),
12027            ),
12028            ("condition_key".to_string(), json!(options.condition_key)),
12029            (
12030                "condition_definition_fingerprint".to_string(),
12031                json!(options.predicate_identity),
12032            ),
12033        ]);
12034        if let Some(timeout_seconds) = options.timeout_seconds {
12035            command.insert("timeout_seconds".to_string(), json!(timeout_seconds));
12036        }
12037        apply_parallel_group_path(&mut command, &self.parallel_group_path);
12038        state.commands.push(Value::Object(command));
12039        drop(state);
12040        self.opened_wait = true;
12041        Poll::Pending
12042    }
12043}
12044
12045fn validate_recorded_condition_wait(
12046    sequence: u64,
12047    recorded_occurrence_id: &str,
12048    recorded_key: Option<&str>,
12049    recorded_predicate_identity: &str,
12050    recorded_timeout_seconds: Option<u64>,
12051    current_occurrence_id: &str,
12052    current: &ValidatedConditionWaitOptions,
12053) -> Result<()> {
12054    if recorded_occurrence_id != current_occurrence_id {
12055        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12056            "condition_wait_occurrence_mismatch",
12057            Some(sequence),
12058            Some(recorded_occurrence_id.to_string()),
12059            Some(current_occurrence_id.to_string()),
12060            "recorded condition occurrence differs from the current authored wait position",
12061        )));
12062    }
12063    if recorded_key != Some(current.condition_key.as_str()) {
12064        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12065            "condition_wait_key_mismatch",
12066            Some(sequence),
12067            recorded_key.map(str::to_string),
12068            Some(current.condition_key.clone()),
12069            "recorded condition identity differs from the current workflow wait",
12070        )));
12071    }
12072    if recorded_predicate_identity != current.predicate_identity {
12073        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12074            "condition_wait_predicate_mismatch",
12075            Some(sequence),
12076            Some(recorded_predicate_identity.to_string()),
12077            Some(current.predicate_identity.clone()),
12078            "recorded condition predicate behavior differs from current workflow code",
12079        )));
12080    }
12081    if recorded_timeout_seconds != current.timeout_seconds {
12082        return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12083            "condition_wait_timeout_mismatch",
12084            Some(sequence),
12085            recorded_timeout_seconds.map(|seconds| format!("{seconds}s")),
12086            current.timeout_seconds.map(|seconds| format!("{seconds}s")),
12087            "recorded condition timeout differs from the current workflow wait",
12088        )));
12089    }
12090    Ok(())
12091}
12092
12093/// Future returned by [`WorkflowContext::start_child_workflow`].
12094pub struct ChildWorkflowCall {
12095    ctx: WorkflowContext,
12096    workflow_type: String,
12097    options: ChildWorkflowOptions,
12098    args: Option<Result<AvroValue>>,
12099    scheduled: bool,
12100    matched_pending: bool,
12101    parallel_group_path: Vec<ParallelGroupMetadata>,
12102}
12103
12104impl ChildWorkflowCall {
12105    fn poll_avro_value(
12106        mut self: Pin<&mut Self>,
12107        _cx: &mut TaskContext<'_>,
12108    ) -> Poll<Result<ChildWorkflowAvroResult>> {
12109        if self.matched_pending {
12110            return Poll::Pending;
12111        }
12112
12113        let ctx = self.ctx.clone();
12114        let mut state = match ctx.state.lock() {
12115            Ok(state) => state,
12116            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12117        };
12118
12119        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12120            let sequence = recorded.sequence();
12121            match recorded {
12122                RecordedCommand::ChildWorkflow {
12123                    workflow_type,
12124                    outcome,
12125                    parallel_group_path,
12126                    ..
12127                } => {
12128                    if let Err(error) = ensure_parallel_path_matches(
12129                        sequence,
12130                        parallel_group_path.as_deref(),
12131                        &self.parallel_group_path,
12132                    ) {
12133                        return Poll::Ready(Err(error));
12134                    }
12135                    if let Some(recorded_type) = workflow_type {
12136                        if recorded_type != self.workflow_type {
12137                            return Poll::Ready(Err(Error::NonDeterministicReplay(
12138                                ReplayFailure::new(
12139                                    "recorded_command_detail_mismatch",
12140                                    Some(sequence),
12141                                    Some(format!("child workflow:{recorded_type}")),
12142                                    Some(format!("child workflow:{}", self.workflow_type)),
12143                                    "recorded child workflow type differs from the current workflow command",
12144                                ),
12145                            )));
12146                        }
12147                    }
12148                    state.command_cursor += 1;
12149                    if let Some(outcome) = outcome {
12150                        return Poll::Ready(outcome.map_err(Error::ChildWorkflowFailed));
12151                    }
12152                    state.matched_recorded_pending = true;
12153                    self.scheduled = true;
12154                    self.matched_pending = true;
12155                    return Poll::Pending;
12156                }
12157                other => {
12158                    return Poll::Ready(Err(command_mismatch(
12159                        &other,
12160                        format!("child workflow:{}", self.workflow_type),
12161                    )));
12162                }
12163            }
12164        }
12165
12166        if !self.scheduled {
12167            if self.options.task_queue.trim().is_empty() {
12168                return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12169                    "task_queue must not be empty".to_string(),
12170                )));
12171            }
12172            for (name, value) in [
12173                (
12174                    "execution_timeout_seconds",
12175                    self.options.execution_timeout_seconds,
12176                ),
12177                ("run_timeout_seconds", self.options.run_timeout_seconds),
12178            ] {
12179                if value == Some(0) {
12180                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(format!(
12181                        "{name} must be at least 1"
12182                    ))));
12183                }
12184            }
12185
12186            let args = match self.args.take().unwrap_or(Ok(AvroValue::Null)) {
12187                Ok(args) => args,
12188                Err(error) => return Poll::Ready(Err(error)),
12189            };
12190            let arguments = match encode_typed_envelope(
12191                &normalize_avro_arguments(args),
12192                &state.payload_codec,
12193            ) {
12194                Ok(arguments) => arguments,
12195                Err(error) => return Poll::Ready(Err(error)),
12196            };
12197            let mut command = json!({
12198                "type": "start_child_workflow",
12199                "workflow_type": self.workflow_type,
12200                "queue": self.options.task_queue,
12201                "parent_close_policy": self.options.parent_close_policy.as_str(),
12202                "arguments": arguments,
12203            });
12204            let object = command
12205                .as_object_mut()
12206                .expect("child workflow command is always an object");
12207            if let Some(policy) = &self.options.retry_policy {
12208                let mut retry_policy = serde_json::Map::new();
12209                if let Some(max_attempts) = policy.max_attempts {
12210                    if max_attempts == 0 {
12211                        return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12212                            "retry_policy.max_attempts must be at least 1".to_string(),
12213                        )));
12214                    }
12215                    retry_policy.insert("max_attempts".to_string(), json!(max_attempts));
12216                }
12217                if !policy.backoff_seconds.is_empty() {
12218                    retry_policy
12219                        .insert("backoff_seconds".to_string(), json!(policy.backoff_seconds));
12220                }
12221                if !policy.non_retryable_error_types.is_empty() {
12222                    retry_policy.insert(
12223                        "non_retryable_error_types".to_string(),
12224                        json!(policy.non_retryable_error_types),
12225                    );
12226                }
12227                if retry_policy.is_empty() {
12228                    return Poll::Ready(Err(Error::InvalidChildWorkflowOptions(
12229                        "retry_policy must configure at least one field".to_string(),
12230                    )));
12231                }
12232                object.insert("retry_policy".to_string(), Value::Object(retry_policy));
12233            }
12234            if let Some(seconds) = self.options.execution_timeout_seconds {
12235                object.insert("execution_timeout_seconds".to_string(), json!(seconds));
12236            }
12237            if let Some(seconds) = self.options.run_timeout_seconds {
12238                object.insert("run_timeout_seconds".to_string(), json!(seconds));
12239            }
12240            apply_parallel_group_path(object, &self.parallel_group_path);
12241            state.commands.push(command);
12242            self.scheduled = true;
12243        }
12244
12245        Poll::Pending
12246    }
12247}
12248
12249impl Future for ChildWorkflowCall {
12250    type Output = Result<ChildWorkflowResult>;
12251
12252    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12253        match self.poll_avro_value(cx) {
12254            Poll::Ready(Ok(result)) => match result.result.into_json() {
12255                Ok(projected) => Poll::Ready(Ok(ChildWorkflowResult {
12256                    parent: result.parent,
12257                    child: result.child,
12258                    child_workflow_type: result.child_workflow_type,
12259                    result: projected,
12260                })),
12261                Err(error) => Poll::Ready(Err(error)),
12262            },
12263            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12264            Poll::Pending => Poll::Pending,
12265        }
12266    }
12267}
12268
12269fn command_mismatch(recorded: &RecordedCommand, actual: impl Into<String>) -> Error {
12270    Error::NonDeterministicReplay(ReplayFailure::new(
12271        "recorded_command_mismatch",
12272        Some(recorded.sequence()),
12273        Some(recorded.shape().to_string()),
12274        Some(actual.into()),
12275        "current workflow command does not match the recorded durable command sequence",
12276    ))
12277}
12278
12279pub struct SignalCall {
12280    ctx: WorkflowContext,
12281    signal_name: String,
12282    runtime_reserved_allowed: bool,
12283    opened_wait: bool,
12284    matched_pending: bool,
12285    parallel_group_path: Vec<ParallelGroupMetadata>,
12286}
12287
12288impl SignalCall {
12289    fn poll_avro_value(
12290        mut self: Pin<&mut Self>,
12291        _cx: &mut TaskContext<'_>,
12292    ) -> Poll<Result<Vec<AvroValue>>> {
12293        if self.matched_pending {
12294            return Poll::Pending;
12295        }
12296        if !self.runtime_reserved_allowed {
12297            if let Err(error) = validate_user_signal_name(&self.signal_name) {
12298                return Poll::Ready(Err(error));
12299            }
12300        }
12301
12302        let ctx = self.ctx.clone();
12303        let mut state = match ctx.state.lock() {
12304            Ok(state) => state,
12305            Err(_) => return Poll::Ready(Err(Error::WorkflowStatePoisoned)),
12306        };
12307
12308        if let Some(recorded) = state.recorded_commands.get(state.command_cursor).cloned() {
12309            match recorded {
12310                RecordedCommand::SignalWait {
12311                    sequence,
12312                    signal_name,
12313                    value,
12314                    parallel_group_path,
12315                } => {
12316                    if let Err(error) = ensure_parallel_path_matches(
12317                        sequence,
12318                        parallel_group_path.as_deref(),
12319                        &self.parallel_group_path,
12320                    ) {
12321                        return Poll::Ready(Err(error));
12322                    }
12323                    if signal_name != self.signal_name {
12324                        return Poll::Ready(Err(Error::NonDeterministicReplay(
12325                            ReplayFailure::new(
12326                                "recorded_command_detail_mismatch",
12327                                Some(sequence),
12328                                Some(format!("signal wait:{signal_name}")),
12329                                Some(format!("signal wait:{}", self.signal_name)),
12330                                "recorded signal name differs from the current workflow command",
12331                            ),
12332                        )));
12333                    }
12334
12335                    state.command_cursor += 1;
12336                    if let Some(value) = value {
12337                        return Poll::Ready(Ok(value));
12338                    }
12339                    if state
12340                        .resume_signal
12341                        .as_ref()
12342                        .is_some_and(|signal| signal.signal_name == self.signal_name)
12343                    {
12344                        let signal = state
12345                            .resume_signal
12346                            .take()
12347                            .expect("matching resume signal is present");
12348                        return Poll::Ready(Ok(signal.arguments));
12349                    }
12350
12351                    state.matched_recorded_pending = true;
12352                    self.opened_wait = true;
12353                    self.matched_pending = true;
12354                    return Poll::Pending;
12355                }
12356                other => {
12357                    return Poll::Ready(Err(command_mismatch(
12358                        &other,
12359                        format!("signal wait:{}", self.signal_name),
12360                    )));
12361                }
12362            }
12363        }
12364
12365        if state
12366            .resume_signal
12367            .as_ref()
12368            .is_some_and(|signal| signal.signal_name == self.signal_name)
12369        {
12370            let signal = state
12371                .resume_signal
12372                .take()
12373                .expect("matching resume signal is present");
12374            return Poll::Ready(Ok(signal.arguments));
12375        }
12376
12377        if !self.opened_wait {
12378            let mut command = serde_json::Map::from_iter([
12379                ("type".to_string(), json!("open_signal_wait")),
12380                ("signal_name".to_string(), json!(self.signal_name)),
12381            ]);
12382            apply_parallel_group_path(&mut command, &self.parallel_group_path);
12383            state.commands.push(Value::Object(command));
12384            self.opened_wait = true;
12385        }
12386
12387        Poll::Pending
12388    }
12389}
12390
12391impl Future for SignalCall {
12392    type Output = Result<Vec<Value>>;
12393
12394    fn poll(self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Self::Output> {
12395        match self.poll_avro_value(cx) {
12396            Poll::Ready(Ok(values)) => Poll::Ready(
12397                values
12398                    .into_iter()
12399                    .map(AvroValue::into_json)
12400                    .collect::<Result<Vec<_>>>(),
12401            ),
12402            Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
12403            Poll::Pending => Poll::Pending,
12404        }
12405    }
12406}
12407
12408#[derive(Clone, Debug)]
12409pub struct ActivityContext {
12410    client: Client,
12411    pub task_id: String,
12412    pub activity_attempt_id: String,
12413    pub lease_owner: String,
12414    pub activity_type: String,
12415    pub attempt_number: u64,
12416    pub task_queue: String,
12417    pub worker_id: String,
12418}
12419
12420impl ActivityContext {
12421    pub async fn heartbeat<T: Serialize>(&self, details: T) -> Result<ActivityHeartbeatResponse> {
12422        self.client
12423            .heartbeat_activity_task(
12424                &self.task_id,
12425                &self.activity_attempt_id,
12426                &self.lease_owner,
12427                details,
12428            )
12429            .await
12430    }
12431}
12432
12433fn decode_task_avro_arguments(value: Option<&Value>, codec: &str) -> Result<AvroValue> {
12434    validate_payload_codec(codec)?;
12435    match value {
12436        Some(value) => Ok(normalize_avro_arguments(decode_wire_avro_value(
12437            value, codec,
12438        )?)),
12439        None => Ok(AvroValue::Array(Vec::new())),
12440    }
12441}
12442
12443fn decode_resume_signal(task: &WorkflowTask) -> Result<Option<ResumeSignal>> {
12444    let Some(signal_name) = task
12445        .signal_name
12446        .as_deref()
12447        .filter(|value| !value.is_empty())
12448    else {
12449        return Ok(None);
12450    };
12451    let decoded = decode_task_avro_arguments(task.signal_arguments.as_ref(), &task.payload_codec)?;
12452    let AvroValue::Array(arguments) = decoded else {
12453        unreachable!("normalize_avro_arguments always returns an array");
12454    };
12455
12456    Ok(Some(ResumeSignal {
12457        signal_name: signal_name.to_string(),
12458        arguments,
12459    }))
12460}
12461
12462fn validate_workflow_task_payloads(task: &WorkflowTask) -> Result<()> {
12463    validate_payload_codec(&task.payload_codec)?;
12464    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)?;
12465    validate_optional_inbound_payload(task.signal_arguments.as_ref(), &task.payload_codec)?;
12466    for event in &task.history_events {
12467        validate_history_event_payloads(event, &task.payload_codec)?;
12468    }
12469    Ok(())
12470}
12471
12472fn validate_activity_task_payloads(task: &ActivityTask) -> Result<()> {
12473    validate_payload_codec(&task.payload_codec)?;
12474    validate_optional_inbound_payload(task.arguments.as_ref(), &task.payload_codec)
12475}
12476
12477fn validate_query_task_payloads(task: &QueryTask) -> Result<()> {
12478    validate_payload_codec(&task.payload_codec)?;
12479    validate_optional_inbound_payload(task.workflow_arguments.as_ref(), &task.payload_codec)?;
12480    validate_optional_inbound_payload(task.query_arguments.as_ref(), &task.payload_codec)?;
12481    for event in &task.history_events {
12482        validate_history_event_payloads(event, &task.payload_codec)?;
12483    }
12484
12485    let Some(export) = task.history_export.as_ref() else {
12486        return Ok(());
12487    };
12488    let export_codec = match export.get("payloads") {
12489        Some(payloads) => declared_payload_codec(payloads, "codec")?,
12490        None => None,
12491    }
12492    .unwrap_or(&task.payload_codec);
12493    validate_payload_codec(export_codec)?;
12494
12495    if let Some(events) = export.get("history_events").and_then(Value::as_array) {
12496        for event in events {
12497            let event_type = event
12498                .get("event_type")
12499                .or_else(|| event.get("type"))
12500                .and_then(Value::as_str)
12501                .unwrap_or_default();
12502            if let Some(payload) = event.get("payload") {
12503                validate_history_payloads(event_type, payload, export_codec)?;
12504            }
12505        }
12506    }
12507    for signal in export
12508        .get("signals")
12509        .and_then(Value::as_array)
12510        .into_iter()
12511        .flatten()
12512    {
12513        let codec = declared_payload_codec(signal, "payload_codec")?.unwrap_or(export_codec);
12514        validate_payload_codec(codec)?;
12515        validate_optional_inbound_payload(signal.get("arguments"), codec)?;
12516    }
12517    for activity in export
12518        .get("activities")
12519        .and_then(Value::as_array)
12520        .into_iter()
12521        .flatten()
12522    {
12523        let codec = declared_payload_codec(activity, "payload_codec")?.unwrap_or(export_codec);
12524        validate_payload_codec(codec)?;
12525        validate_optional_inbound_payload(activity.get("arguments"), codec)?;
12526        validate_optional_inbound_payload(activity.get("result"), codec)?;
12527    }
12528    Ok(())
12529}
12530
12531fn validate_history_event_payloads(event: &HistoryEvent, fallback_codec: &str) -> Result<()> {
12532    validate_history_payloads(&event.event_type, &event.payload, fallback_codec)
12533}
12534
12535fn validate_history_payloads(
12536    event_type: &str,
12537    payload: &Value,
12538    fallback_codec: &str,
12539) -> Result<()> {
12540    let codec = declared_payload_codec(payload, "payload_codec")?.unwrap_or(fallback_codec);
12541    validate_payload_codec(codec)?;
12542    for field in history_payload_fields(event_type) {
12543        validate_optional_inbound_payload(payload.get(*field), codec)?;
12544    }
12545    Ok(())
12546}
12547
12548const SIGNAL_HISTORY_PAYLOAD_FIELDS: &[&str] = &["value", "input", "arguments"];
12549
12550fn history_payload_fields(event_type: &str) -> &'static [&'static str] {
12551    match event_type {
12552        "ActivityCompleted" => &["result"],
12553        "SignalReceived" | "SignalApplied" => SIGNAL_HISTORY_PAYLOAD_FIELDS,
12554        "UpdateAccepted" | "UpdateRejected" | "UpdateApplied" => &["arguments"],
12555        "UpdateCompleted" | "SideEffectRecorded" => &["result"],
12556        "ChildRunCompleted" => &["result", "output"],
12557        "WorkflowCompleted" => &["output"],
12558        "ServiceCallStarted"
12559        | "ServiceCallCompleted"
12560        | "ServiceCallFailed"
12561        | "ServiceCallCancelled" => &["request_payload", "response_payload"],
12562        _ => &[],
12563    }
12564}
12565
12566fn signal_history_payload(payload: &Value) -> Option<&Value> {
12567    SIGNAL_HISTORY_PAYLOAD_FIELDS
12568        .iter()
12569        .find_map(|field| payload.get(*field))
12570}
12571
12572fn declared_payload_codec<'a>(value: &'a Value, field: &str) -> Result<Option<&'a str>> {
12573    match value.get(field) {
12574        None => Ok(None),
12575        Some(Value::String(codec)) => Ok(Some(codec)),
12576        Some(_) => Err(invalid_payload_envelope()),
12577    }
12578}
12579
12580fn validate_optional_inbound_payload(value: Option<&Value>, codec: &str) -> Result<()> {
12581    validate_payload_codec(codec)?;
12582    if let Some(value) = value.filter(|value| !value.is_null()) {
12583        decode_wire_avro_value(value, codec)?;
12584    }
12585    Ok(())
12586}
12587
12588fn recorded_parallel_group_entry(payload: &Value, sequence: u64) -> Result<ParallelGroupMetadata> {
12589    let group_id = payload_string(payload, "parallel_group_id").ok_or_else(|| {
12590        invalid_recorded_history(
12591            "parallel_group_metadata_invalid",
12592            sequence,
12593            "non-empty parallel_group_id",
12594            &payload.to_string(),
12595            "parallel-group history is missing its stable identity",
12596        )
12597    })?;
12598    let kind = payload_string(payload, "parallel_group_kind").ok_or_else(|| {
12599        invalid_recorded_history(
12600            "parallel_group_metadata_invalid",
12601            sequence,
12602            "activity, child, timer, signal, condition, or mixed group kind",
12603            &payload.to_string(),
12604            "parallel-group history is missing its group kind",
12605        )
12606    })?;
12607    if !matches!(
12608        kind.as_str(),
12609        "activity" | "child" | "timer" | "signal" | "condition" | "mixed"
12610    ) {
12611        return Err(invalid_recorded_history(
12612            "parallel_group_metadata_invalid",
12613            sequence,
12614            "activity, child, timer, signal, condition, or mixed group kind",
12615            &kind,
12616            "parallel-group history contains an unsupported group kind",
12617        ));
12618    }
12619    let base_sequence = payload
12620        .get("parallel_group_base_sequence")
12621        .and_then(value_as_u64)
12622        .filter(|value| *value > 0)
12623        .ok_or_else(|| {
12624            invalid_recorded_history(
12625                "parallel_group_metadata_invalid",
12626                sequence,
12627                "positive parallel_group_base_sequence",
12628                &payload.to_string(),
12629                "parallel-group history contains an invalid base sequence",
12630            )
12631        })?;
12632    let size = payload
12633        .get("parallel_group_size")
12634        .and_then(value_as_u64)
12635        .and_then(|value| usize::try_from(value).ok())
12636        .filter(|value| (1..=MAX_PARALLEL_OPERATIONS).contains(value))
12637        .ok_or_else(|| {
12638            invalid_recorded_history(
12639                "parallel_group_metadata_invalid",
12640                sequence,
12641                "bounded positive parallel_group_size",
12642                &payload.to_string(),
12643                "parallel-group history contains an invalid group size",
12644            )
12645        })?;
12646    let index = payload
12647        .get("parallel_group_index")
12648        .and_then(value_as_u64)
12649        .and_then(|value| usize::try_from(value).ok())
12650        .filter(|value| *value < size)
12651        .ok_or_else(|| {
12652            invalid_recorded_history(
12653                "parallel_group_metadata_invalid",
12654                sequence,
12655                "parallel_group_index within group bounds",
12656                &payload.to_string(),
12657                "parallel-group history contains an invalid member index",
12658            )
12659        })?;
12660    if base_sequence.checked_add(u64::try_from(index).unwrap_or(u64::MAX)) != Some(sequence) {
12661        return Err(invalid_recorded_history(
12662            "parallel_group_metadata_invalid",
12663            sequence,
12664            "base sequence plus member index equals workflow sequence",
12665            &payload.to_string(),
12666            "parallel-group path does not preserve durable workflow position",
12667        ));
12668    }
12669    let mode = payload
12670        .get("parallel_group_mode")
12671        .and_then(Value::as_str)
12672        .unwrap_or("all");
12673    if !matches!(mode, "all" | "select") {
12674        return Err(invalid_recorded_history(
12675            "parallel_group_metadata_invalid",
12676            sequence,
12677            "parallel group mode all or select",
12678            mode,
12679            "parallel-group history contains an unsupported group mode",
12680        ));
12681    }
12682    let expected_id = if mode == "select" {
12683        format!("select-calls:{base_sequence}:{size}")
12684    } else {
12685        format!("{}:{base_sequence}:{size}", parallel_group_prefix(&kind))
12686    };
12687    if group_id != expected_id {
12688        return Err(invalid_recorded_history(
12689            "parallel_group_metadata_invalid",
12690            sequence,
12691            &expected_id,
12692            &group_id,
12693            "parallel-group history contains an incompatible stable group ID",
12694        ));
12695    }
12696    let selection_member_key = if mode == "select" {
12697        Some(selection_key_from_value(
12698            payload.get("selection_member_key"),
12699            sequence,
12700        )?)
12701    } else {
12702        None
12703    };
12704    let selection_member_index = if mode == "select" {
12705        Some(required_parallel_usize(
12706            payload,
12707            "selection_member_index",
12708            sequence,
12709        )?)
12710    } else {
12711        None
12712    };
12713    let selection_member_base_sequence = if mode == "select" {
12714        Some(
12715            payload
12716                .get("selection_member_base_sequence")
12717                .and_then(value_as_u64)
12718                .filter(|value| *value >= base_sequence)
12719                .ok_or_else(|| {
12720                    invalid_recorded_history(
12721                        "parallel_group_metadata_invalid",
12722                        sequence,
12723                        "selection member base within its group",
12724                        &payload.to_string(),
12725                        "selection history contains an invalid member base sequence",
12726                    )
12727                })?,
12728        )
12729    } else {
12730        None
12731    };
12732    let selection_member_size = if mode == "select" {
12733        let member_size = required_parallel_usize(payload, "selection_member_size", sequence)?;
12734        if member_size == 0 {
12735            return Err(invalid_recorded_history(
12736                "parallel_group_metadata_invalid",
12737                sequence,
12738                "positive selection member size",
12739                &payload.to_string(),
12740                "selection history contains an invalid member size",
12741            ));
12742        }
12743        Some(member_size)
12744    } else {
12745        None
12746    };
12747    let selection_member_kind = if mode == "select" {
12748        let kind = payload_string(payload, "selection_member_kind").ok_or_else(|| {
12749            invalid_recorded_history(
12750                "parallel_group_metadata_invalid",
12751                sequence,
12752                "selection member operation kind",
12753                &payload.to_string(),
12754                "selection history is missing its authored member kind",
12755            )
12756        })?;
12757        if !matches!(
12758            kind.as_str(),
12759            "activity" | "child" | "timer" | "signal" | "condition" | "group"
12760        ) {
12761            return Err(invalid_recorded_history(
12762                "parallel_group_metadata_invalid",
12763                sequence,
12764                "activity, child, timer, signal, condition, or group selection member kind",
12765                &kind,
12766                "selection history contains an unsupported member kind",
12767            ));
12768        }
12769        Some(kind)
12770    } else {
12771        None
12772    };
12773    if let (Some(member_base), Some(member_size)) =
12774        (selection_member_base_sequence, selection_member_size)
12775    {
12776        let member_end = member_base
12777            .checked_add(u64::try_from(member_size).unwrap_or(u64::MAX))
12778            .ok_or_else(|| {
12779                invalid_recorded_history(
12780                    "parallel_group_metadata_invalid",
12781                    sequence,
12782                    "bounded selection member range",
12783                    &payload.to_string(),
12784                    "selection member range overflowed",
12785                )
12786            })?;
12787        let group_end = base_sequence
12788            .checked_add(u64::try_from(size).unwrap_or(u64::MAX))
12789            .unwrap_or(u64::MAX);
12790        if sequence < member_base || sequence >= member_end || member_end > group_end {
12791            return Err(invalid_recorded_history(
12792                "parallel_group_metadata_invalid",
12793                sequence,
12794                "workflow sequence within one bounded selection member",
12795                &payload.to_string(),
12796                "selection member range does not contain its durable leaf",
12797            ));
12798        }
12799    }
12800    Ok(ParallelGroupMetadata {
12801        parallel_group_id: group_id,
12802        parallel_group_kind: kind,
12803        parallel_group_base_sequence: base_sequence,
12804        parallel_group_size: size,
12805        parallel_group_index: index,
12806        parallel_group_mode: (mode == "select").then(|| "select".to_string()),
12807        selection_member_key,
12808        selection_member_index,
12809        selection_member_base_sequence,
12810        selection_member_size,
12811        selection_member_kind,
12812    })
12813}
12814
12815fn required_parallel_usize(payload: &Value, field: &str, sequence: u64) -> Result<usize> {
12816    payload
12817        .get(field)
12818        .and_then(value_as_u64)
12819        .and_then(|value| usize::try_from(value).ok())
12820        .ok_or_else(|| {
12821            invalid_recorded_history(
12822                "parallel_group_metadata_invalid",
12823                sequence,
12824                &format!("non-negative integer {field}"),
12825                &payload.to_string(),
12826                "selection history contains invalid member metadata",
12827            )
12828        })
12829}
12830
12831fn selection_key_from_value(value: Option<&Value>, sequence: u64) -> Result<SelectionKey> {
12832    match value {
12833        Some(Value::String(value)) if !value.is_empty() => Ok(SelectionKey::Name(value.clone())),
12834        Some(value) => value_as_u64(value)
12835            .and_then(|value| usize::try_from(value).ok())
12836            .map(SelectionKey::Index)
12837            .ok_or_else(|| {
12838                invalid_recorded_history(
12839                    "selection_member_key_invalid",
12840                    sequence,
12841                    "non-empty string or non-negative integer member key",
12842                    &value.to_string(),
12843                    "selection history contains an invalid member key",
12844                )
12845            }),
12846        None => Err(invalid_recorded_history(
12847            "selection_member_key_missing",
12848            sequence,
12849            "selection_member_key",
12850            "<missing>",
12851            "selection history is missing its stable member key",
12852        )),
12853    }
12854}
12855
12856fn recorded_parallel_group_path(
12857    events: &[&HistoryEvent],
12858    sequence: u64,
12859) -> Result<Option<Vec<ParallelGroupMetadata>>> {
12860    let mut recorded: Option<Vec<ParallelGroupMetadata>> = None;
12861    for event in events {
12862        let payload = &event.payload;
12863        let has_metadata = payload.get("parallel_group_path").is_some()
12864            || payload.get("parallel_group_id").is_some()
12865            || payload.get("parallel_group_kind").is_some()
12866            || payload.get("parallel_group_base_sequence").is_some()
12867            || payload.get("parallel_group_size").is_some()
12868            || payload.get("parallel_group_index").is_some()
12869            || payload.get("parallel_group_mode").is_some()
12870            || payload.get("selection_member_key").is_some();
12871        if !has_metadata {
12872            continue;
12873        }
12874
12875        let top_level = recorded_parallel_group_entry(payload, sequence)?;
12876        let path = match payload.get("parallel_group_path") {
12877            None => vec![top_level.clone()],
12878            Some(Value::Array(entries)) if !entries.is_empty() => entries
12879                .iter()
12880                .map(|entry| recorded_parallel_group_entry(entry, sequence))
12881                .collect::<Result<Vec<_>>>()?,
12882            Some(value) => {
12883                return Err(invalid_recorded_history(
12884                    "parallel_group_metadata_invalid",
12885                    sequence,
12886                    "non-empty parallel_group_path list",
12887                    &value.to_string(),
12888                    "parallel-group history contains an invalid group path",
12889                ));
12890            }
12891        };
12892        if path.last() != Some(&top_level) {
12893            return Err(invalid_recorded_history(
12894                "parallel_group_metadata_invalid",
12895                sequence,
12896                &serde_json::to_string(&path.last()).unwrap_or_default(),
12897                &serde_json::to_string(&top_level).unwrap_or_default(),
12898                "parallel-group top-level fields do not match the innermost path entry",
12899            ));
12900        }
12901        if recorded.as_ref().is_some_and(|existing| existing != &path) {
12902            return Err(invalid_recorded_history(
12903                "parallel_group_history_conflict",
12904                sequence,
12905                &serde_json::to_string(&recorded.as_ref()).unwrap_or_default(),
12906                &serde_json::to_string(&path).unwrap_or_default(),
12907                "parallel-group metadata changed between scheduling and resolution history",
12908            ));
12909        }
12910        recorded = Some(path);
12911    }
12912    Ok(recorded)
12913}
12914
12915fn recorded_commands(
12916    events: &[HistoryEvent],
12917    fallback_codec: &str,
12918    parent: WorkflowIdentity,
12919) -> Result<Vec<RecordedCommand>> {
12920    let mut events_by_sequence: BTreeMap<u64, Vec<&HistoryEvent>> = BTreeMap::new();
12921    let mut last_new_sequence = None;
12922
12923    for event in events {
12924        let is_activity = matches!(
12925            event.event_type.as_str(),
12926            "ActivityScheduled"
12927                | "ActivityStarted"
12928                | "ActivityHeartbeatRecorded"
12929                | "ActivityRetryScheduled"
12930                | "ActivityCompleted"
12931                | "ActivityFailed"
12932                | "ActivityCancelled"
12933                | "ActivityTimedOut"
12934        );
12935        let is_workflow_timer = matches!(
12936            event.event_type.as_str(),
12937            "TimerScheduled" | "TimerCancelled" | "TimerFired"
12938        ) && !is_internal_timer_event(event);
12939        let is_child_workflow = matches!(
12940            event.event_type.as_str(),
12941            "ChildWorkflowScheduled"
12942                | "ChildRunCompleted"
12943                | "ChildRunFailed"
12944                | "ChildRunCancelled"
12945                | "ChildRunTerminated"
12946        );
12947        let is_signal_wait = is_recorded_signal_wait_event(event);
12948        let is_condition_wait = is_recorded_condition_wait_event(event);
12949        let is_search_attributes = event.event_type == "SearchAttributesUpserted";
12950        let is_side_effect = event.event_type == "SideEffectRecorded";
12951        let is_version_marker = event.event_type == "VersionMarkerRecorded";
12952        let is_memo = event.event_type == "MemoUpserted";
12953        if !is_activity
12954            && !is_workflow_timer
12955            && !is_child_workflow
12956            && !is_signal_wait
12957            && !is_condition_wait
12958            && !is_search_attributes
12959            && !is_side_effect
12960            && !is_version_marker
12961            && !is_memo
12962        {
12963            continue;
12964        }
12965
12966        let sequence = durable_event_sequence(event).ok_or_else(|| {
12967            Error::NonDeterministicReplay(ReplayFailure::new(
12968                "durable_command_sequence_missing",
12969                None,
12970                Some("positive workflow sequence".to_string()),
12971                Some(event.event_type.clone()),
12972                "durable command history event has no workflow sequence",
12973            ))
12974        })?;
12975        if sequence == 0 {
12976            return Err(Error::NonDeterministicReplay(ReplayFailure::new(
12977                "durable_command_sequence_invalid",
12978                Some(sequence),
12979                Some("positive workflow sequence".to_string()),
12980                Some(sequence.to_string()),
12981                "durable command history uses an invalid workflow sequence",
12982            )));
12983        }
12984        if !events_by_sequence.contains_key(&sequence) {
12985            if let Some(previous) = last_new_sequence {
12986                if sequence < previous {
12987                    return Err(invalid_recorded_history(
12988                        "durable_command_sequence_mismatch",
12989                        sequence,
12990                        &format!("workflow sequence greater than {previous}"),
12991                        &sequence.to_string(),
12992                        "durable commands are not strictly ordered by their recorded workflow sequence",
12993                    ));
12994                }
12995            }
12996            last_new_sequence = Some(sequence);
12997        }
12998        events_by_sequence.entry(sequence).or_default().push(event);
12999    }
13000
13001    let commands: Vec<RecordedCommand> = events_by_sequence
13002        .into_iter()
13003        .map(|(sequence, sequence_events)| {
13004            let activity_events: Vec<_> = sequence_events
13005                .iter()
13006                .copied()
13007                .filter(|event| event.event_type.starts_with("Activity"))
13008                .collect();
13009            let timer_events: Vec<_> = sequence_events
13010                .iter()
13011                .copied()
13012                .filter(|event| event.event_type.starts_with("Timer"))
13013                .collect();
13014            let child_events: Vec<_> = sequence_events
13015                .iter()
13016                .copied()
13017                .filter(|event| {
13018                    event.event_type == "ChildWorkflowScheduled"
13019                        || event.event_type.starts_with("ChildRun")
13020                })
13021                .collect();
13022            let signal_wait_events: Vec<_> = sequence_events
13023                .iter()
13024                .copied()
13025                .filter(|event| is_recorded_signal_wait_event(event))
13026                .collect();
13027            let condition_wait_events: Vec<_> = sequence_events
13028                .iter()
13029                .copied()
13030                .filter(|event| is_recorded_condition_wait_event(event))
13031                .collect();
13032            let search_attribute_events: Vec<_> = sequence_events
13033                .iter()
13034                .copied()
13035                .filter(|event| event.event_type == "SearchAttributesUpserted")
13036                .collect();
13037            let side_effect_events: Vec<_> = sequence_events
13038                .iter()
13039                .copied()
13040                .filter(|event| event.event_type == "SideEffectRecorded")
13041                .collect();
13042            let version_marker_events: Vec<_> = sequence_events
13043                .iter()
13044                .copied()
13045                .filter(|event| event.event_type == "VersionMarkerRecorded")
13046                .collect();
13047            let memo_events: Vec<_> = sequence_events
13048                .iter()
13049                .copied()
13050                .filter(|event| event.event_type == "MemoUpserted")
13051                .collect();
13052
13053            let command_kind_count = usize::from(!activity_events.is_empty())
13054                + usize::from(!timer_events.is_empty())
13055                + usize::from(!child_events.is_empty())
13056                + usize::from(!signal_wait_events.is_empty())
13057                + usize::from(!condition_wait_events.is_empty())
13058                + usize::from(!search_attribute_events.is_empty())
13059                + usize::from(!side_effect_events.is_empty())
13060                + usize::from(!version_marker_events.is_empty())
13061                + usize::from(!memo_events.is_empty());
13062            if command_kind_count > 1 {
13063                let actual = [
13064                    (!activity_events.is_empty()).then_some("activity"),
13065                    (!timer_events.is_empty()).then_some("timer"),
13066                    (!child_events.is_empty()).then_some("child workflow"),
13067                    (!signal_wait_events.is_empty()).then_some("signal wait"),
13068                    (!condition_wait_events.is_empty()).then_some("condition wait"),
13069                    (!search_attribute_events.is_empty()).then_some("search-attribute update"),
13070                    (!side_effect_events.is_empty()).then_some("side effect"),
13071                    (!version_marker_events.is_empty()).then_some("version marker"),
13072                    (!memo_events.is_empty()).then_some("memo upsert"),
13073                ]
13074                .into_iter()
13075                .flatten()
13076                .collect::<Vec<_>>()
13077                .join(" and ");
13078                return Err(invalid_recorded_history(
13079                    "durable_command_sequence_collision",
13080                    sequence,
13081                    "one durable command kind",
13082                    &actual,
13083                    "one workflow sequence records more than one durable command kind",
13084                ));
13085            }
13086
13087            if !activity_events.is_empty() {
13088                let parallel_group_path =
13089                    recorded_parallel_group_path(&activity_events, sequence)?;
13090                let scheduled_count = activity_events
13091                    .iter()
13092                    .filter(|event| event.event_type == "ActivityScheduled")
13093                    .count();
13094                if scheduled_count > 1 {
13095                    return Err(invalid_recorded_history(
13096                        "duplicate_activity_schedule",
13097                        sequence,
13098                        "at most one ActivityScheduled event",
13099                        "multiple ActivityScheduled events",
13100                        "activity history schedules more than one command at one workflow sequence",
13101                    ));
13102                }
13103                let activity_type = activity_events.iter().find_map(|event| {
13104                    event
13105                        .payload
13106                        .get("activity_type")
13107                        .or_else(|| event.payload.get("activity_name"))
13108                        .and_then(Value::as_str)
13109                        .map(str::to_string)
13110                });
13111                if activity_events.iter().filter_map(|event| {
13112                    event
13113                        .payload
13114                        .get("activity_type")
13115                        .or_else(|| event.payload.get("activity_name"))
13116                        .and_then(Value::as_str)
13117                }).any(|candidate| Some(candidate) != activity_type.as_deref()) {
13118                    return Err(invalid_recorded_history(
13119                        "activity_identity_mismatch",
13120                        sequence,
13121                        activity_type.as_deref().unwrap_or("one activity identity"),
13122                        "conflicting activity identities",
13123                        "activity lifecycle events at one workflow sequence disagree on identity",
13124                    ));
13125                }
13126                let terminal: Vec<_> = activity_events
13127                    .iter()
13128                    .copied()
13129                    .filter(|event| {
13130                        matches!(
13131                            event.event_type.as_str(),
13132                            "ActivityCompleted"
13133                                | "ActivityFailed"
13134                                | "ActivityCancelled"
13135                                | "ActivityTimedOut"
13136                        )
13137                    })
13138                    .collect();
13139                let duplicate_delivery = terminal.first().is_some_and(|first| {
13140                    terminal.iter().all(|event| {
13141                        event.event_type == first.event_type && event.payload == first.payload
13142                    })
13143                });
13144                if terminal.len() > 1 && !duplicate_delivery {
13145                    return Err(invalid_recorded_history(
13146                        "duplicate_activity_terminal_event",
13147                        sequence,
13148                        "at most one terminal activity event",
13149                        "multiple terminal activity events",
13150                        "activity history settles one command more than once",
13151                    ));
13152                }
13153                let outcome = terminal
13154                    .first()
13155                    .map(|event| activity_outcome(event, fallback_codec, activity_type.clone()))
13156                    .transpose()?;
13157                let options = activity_events
13158                    .iter()
13159                    .find(|event| event.event_type == "ActivityScheduled")
13160                    .and_then(|event| event.payload.get("activity"))
13161                    .and_then(Value::as_object)
13162                    .map(|activity| RecordedActivityOptions {
13163                        task_queue: recorded_optional_string(activity, "queue"),
13164                        execution_mode: recorded_optional_string(activity, "execution_mode"),
13165                        retry_policy: recorded_activity_retry_snapshot(
13166                            activity.get("retry_policy"),
13167                        ),
13168                    });
13169                return Ok(RecordedCommand::Activity {
13170                    sequence,
13171                    activity_type,
13172                    options,
13173                    outcome,
13174                    parallel_group_path,
13175                });
13176            }
13177
13178            if !child_events.is_empty() {
13179                let parallel_group_path = recorded_parallel_group_path(&child_events, sequence)?;
13180                let scheduled: Vec<_> = child_events
13181                    .iter()
13182                    .copied()
13183                    .filter(|event| event.event_type == "ChildWorkflowScheduled")
13184                    .collect();
13185                if scheduled.len() != 1 {
13186                    return Err(invalid_recorded_history(
13187                        "child_workflow_schedule_missing_or_duplicate",
13188                        sequence,
13189                        "one ChildWorkflowScheduled event",
13190                        &format!("{} ChildWorkflowScheduled events", scheduled.len()),
13191                        "child workflow replay requires exactly one recorded schedule event",
13192                    ));
13193                }
13194                let workflow_type = child_events.iter().find_map(|event| {
13195                    event
13196                        .payload
13197                        .get("child_workflow_type")
13198                        .or_else(|| event.payload.get("workflow_type"))
13199                        .and_then(Value::as_str)
13200                        .filter(|value| !value.is_empty())
13201                        .map(str::to_string)
13202                });
13203                if child_events
13204                    .iter()
13205                    .filter_map(|event| {
13206                        event
13207                            .payload
13208                            .get("child_workflow_type")
13209                            .or_else(|| event.payload.get("workflow_type"))
13210                            .and_then(Value::as_str)
13211                    })
13212                    .any(|candidate| Some(candidate) != workflow_type.as_deref())
13213                {
13214                    return Err(invalid_recorded_history(
13215                        "child_workflow_identity_mismatch",
13216                        sequence,
13217                        workflow_type
13218                            .as_deref()
13219                            .unwrap_or("one child workflow type"),
13220                        "conflicting child workflow types",
13221                        "child workflow lifecycle events at one sequence disagree on type",
13222                    ));
13223                }
13224                let mut outcomes = child_workflow_outcomes(
13225                    &child_events.iter().map(|event| (*event).clone()).collect::<Vec<_>>(),
13226                    fallback_codec,
13227                    parent.clone(),
13228                )?;
13229                let terminal_events = child_events
13230                    .iter()
13231                    .copied()
13232                    .filter(|event| event.event_type.starts_with("ChildRun"))
13233                    .collect::<Vec<_>>();
13234                let duplicate_delivery = terminal_events.first().is_some_and(|first| {
13235                    terminal_events.iter().all(|event| {
13236                        event.event_type == first.event_type && event.payload == first.payload
13237                    })
13238                });
13239                if outcomes.len() > 1 && !duplicate_delivery {
13240                    return Err(invalid_recorded_history(
13241                        "duplicate_child_workflow_terminal_event",
13242                        sequence,
13243                        "at most one terminal child event",
13244                        "multiple terminal child events",
13245                        "child workflow history settles one command more than once",
13246                    ));
13247                }
13248                return Ok(RecordedCommand::ChildWorkflow {
13249                    sequence,
13250                    workflow_type,
13251                    outcome: outcomes.pop(),
13252                    parallel_group_path,
13253                });
13254            }
13255
13256            if !signal_wait_events.is_empty() {
13257                let opened: Vec<_> = signal_wait_events
13258                    .iter()
13259                    .copied()
13260                    .filter(|event| event.event_type == "SignalWaitOpened")
13261                    .collect();
13262                if opened.len() != 1 {
13263                    return Err(invalid_recorded_history(
13264                        "signal_wait_open_missing_or_duplicate",
13265                        sequence,
13266                        "one SignalWaitOpened event",
13267                        &format!("{} SignalWaitOpened events", opened.len()),
13268                        "signal replay requires exactly one canonical wait-open event",
13269                    ));
13270                }
13271
13272                let applied: Vec<_> = signal_wait_events
13273                    .iter()
13274                    .copied()
13275                    .filter(|event| event.event_type == "SignalApplied")
13276                    .collect();
13277                if applied.len() > 1 {
13278                    return Err(invalid_recorded_history(
13279                        "duplicate_signal_wait_apply",
13280                        sequence,
13281                        "at most one SignalApplied event",
13282                        "multiple SignalApplied events",
13283                        "signal history applies one durable wait more than once",
13284                    ));
13285                }
13286
13287                let signal_names = signal_wait_events
13288                    .iter()
13289                    .map(|event| required_signal_wait_name(event, sequence))
13290                    .collect::<Result<Vec<_>>>()?;
13291                let signal_name = signal_names
13292                    .first()
13293                    .expect("signal wait events are not empty")
13294                    .clone();
13295                if signal_names.iter().any(|candidate| candidate != &signal_name) {
13296                    return Err(invalid_recorded_history(
13297                        "signal_wait_identity_mismatch",
13298                        sequence,
13299                        &signal_name,
13300                        "conflicting signal names",
13301                        "signal wait lifecycle events at one workflow sequence disagree on identity",
13302                    ));
13303                }
13304                let value = applied
13305                    .first()
13306                    .map(|event| decode_signal_event_arguments(event, fallback_codec))
13307                    .transpose()?;
13308                return Ok(RecordedCommand::SignalWait {
13309                    sequence,
13310                    signal_name,
13311                    value,
13312                    parallel_group_path: recorded_parallel_group_path(
13313                        &signal_wait_events,
13314                        sequence,
13315                    )?,
13316                });
13317            }
13318
13319            if !condition_wait_events.is_empty() {
13320                return recorded_condition_wait(
13321                    sequence,
13322                    &condition_wait_events,
13323                    events,
13324                );
13325            }
13326
13327            if !search_attribute_events.is_empty() {
13328                if search_attribute_events.len() != 1 {
13329                    return Err(invalid_recorded_history(
13330                        "duplicate_search_attribute_update",
13331                        sequence,
13332                        "one SearchAttributesUpserted event",
13333                        &format!(
13334                            "{} SearchAttributesUpserted events",
13335                            search_attribute_events.len()
13336                        ),
13337                        "search-attribute history records one workflow command more than once",
13338                    ));
13339                }
13340                let payload = &search_attribute_events[0].payload;
13341                let attributes = payload
13342                    .get("attributes")
13343                    .filter(|value| value.as_object().is_some_and(|values| !values.is_empty()))
13344                    .cloned()
13345                    .ok_or_else(|| {
13346                        invalid_recorded_history(
13347                            "search_attribute_update_missing",
13348                            sequence,
13349                            "non-empty attributes object",
13350                            "missing or invalid attributes",
13351                            "search-attribute history is missing its recorded mutation",
13352                        )
13353                    })?;
13354                let attribute_types =
13355                    recorded_search_attribute_types(payload, &attributes, sequence)?;
13356                return Ok(RecordedCommand::SearchAttributes {
13357                    sequence,
13358                    attributes,
13359                    attribute_types,
13360                });
13361            }
13362
13363            if !side_effect_events.is_empty() {
13364                if side_effect_events.len() != 1 {
13365                    return Err(invalid_recorded_history(
13366                        "duplicate_side_effect_record",
13367                        sequence,
13368                        "one SideEffectRecorded event",
13369                        &format!("{} SideEffectRecorded events", side_effect_events.len()),
13370                        "side-effect history records one workflow command more than once",
13371                    ));
13372                }
13373                let event = side_effect_events[0];
13374                let result = event.payload.get("result").ok_or_else(|| {
13375                    invalid_recorded_history(
13376                        "side_effect_result_missing",
13377                        sequence,
13378                        "recorded result payload",
13379                        "missing result",
13380                        "side-effect history is missing its recorded value",
13381                    )
13382                })?;
13383                let has_published_envelope = result.as_str().is_some()
13384                    || result.as_object().is_some_and(|envelope| {
13385                        envelope.get("codec").and_then(Value::as_str).is_some()
13386                            && envelope.get("blob").and_then(Value::as_str).is_some()
13387                    });
13388                if !has_published_envelope {
13389                    return Err(invalid_recorded_history(
13390                        "side_effect_payload_malformed",
13391                        sequence,
13392                        "payload blob or {codec, blob} envelope",
13393                        &result.to_string(),
13394                        "side-effect history result does not use a published payload envelope",
13395                    ));
13396                }
13397                let codec = event
13398                    .payload
13399                    .get("payload_codec")
13400                    .and_then(Value::as_str)
13401                    .unwrap_or(fallback_codec);
13402                let value = decode_wire_avro_value(result, codec).map_err(|error| {
13403                    if error.to_string().contains("unsupported_payload_codec") {
13404                        return error;
13405                    }
13406
13407                    invalid_recorded_history(
13408                        "side_effect_payload_incompatible",
13409                        sequence,
13410                        &format!("valid {codec} payload envelope"),
13411                        &error.to_string(),
13412                        "side-effect history payload cannot be decoded with its recorded codec",
13413                    )
13414                })?;
13415                return Ok(RecordedCommand::SideEffect { sequence, value });
13416            }
13417
13418            if !version_marker_events.is_empty() {
13419                if version_marker_events.len() != 1 {
13420                    return Err(invalid_recorded_history(
13421                        "duplicate_version_marker_record",
13422                        sequence,
13423                        "one VersionMarkerRecorded event",
13424                        &format!("{} VersionMarkerRecorded events", version_marker_events.len()),
13425                        "version-marker history records one workflow command more than once",
13426                    ));
13427                }
13428                let payload = &version_marker_events[0].payload;
13429                let change_id = payload
13430                    .get("change_id")
13431                    .and_then(Value::as_str)
13432                    .filter(|value| !value.is_empty())
13433                    .map(str::to_string)
13434                    .ok_or_else(|| {
13435                        invalid_recorded_history(
13436                            "version_marker_field_missing",
13437                            sequence,
13438                            "non-empty change_id",
13439                            "missing or invalid change_id",
13440                            "version-marker history is missing its stable change ID",
13441                        )
13442                    })?;
13443                let version = required_version_i32(payload, "version", sequence)?;
13444                let min_supported = required_version_i32(payload, "min_supported", sequence)?;
13445                let max_supported = required_version_i32(payload, "max_supported", sequence)?;
13446                if min_supported > max_supported || version < min_supported || version > max_supported {
13447                    return Err(invalid_recorded_history(
13448                        "version_marker_history_range_invalid",
13449                        sequence,
13450                        "min_supported <= version <= max_supported",
13451                        &format!("{min_supported} <= {version} <= {max_supported}"),
13452                        "recorded version marker contains an internally incompatible range",
13453                    ));
13454                }
13455                return Ok(RecordedCommand::VersionMarker {
13456                    sequence,
13457                    change_id,
13458                    version,
13459                });
13460            }
13461
13462            if !memo_events.is_empty() {
13463                if memo_events.len() != 1 {
13464                    return Err(invalid_recorded_history(
13465                        "duplicate_memo_upsert_record",
13466                        sequence,
13467                        "one MemoUpserted event",
13468                        &format!("{} MemoUpserted events", memo_events.len()),
13469                        "memo history records one workflow update more than once",
13470                    ));
13471                }
13472                let payload = &memo_events[0].payload;
13473                let entries = payload.get("entries").cloned().ok_or_else(|| {
13474                    invalid_recorded_history(
13475                        "memo_entries_missing",
13476                        sequence,
13477                        "memo entries object",
13478                        "missing entries",
13479                        "MemoUpserted history is missing replay identity entries",
13480                    )
13481                })?;
13482                let entries = decode_memo_history_map(&entries, true).map_err(|error| {
13483                    invalid_recorded_history(
13484                        "memo_entries_invalid",
13485                        sequence,
13486                        "valid canonical memo entries",
13487                        &error.to_string(),
13488                        "MemoUpserted history contains invalid replay identity entries",
13489                    )
13490                })?;
13491                let merged = payload.get("merged").cloned().ok_or_else(|| {
13492                    invalid_recorded_history(
13493                        "memo_merged_projection_missing",
13494                        sequence,
13495                        "merged memo projection",
13496                        "missing merged",
13497                        "MemoUpserted history is missing its merged projection",
13498                    )
13499                })?;
13500                decode_memo_history_map(&merged, false).map_err(|error| {
13501                    invalid_recorded_history(
13502                        "memo_merged_projection_invalid",
13503                        sequence,
13504                        "valid merged memo projection",
13505                        &error.to_string(),
13506                        "MemoUpserted history contains an invalid merged projection",
13507                    )
13508                })?;
13509
13510                return Ok(RecordedCommand::Memo { sequence, entries });
13511            }
13512            let scheduled: Vec<_> = timer_events
13513                .iter()
13514                .copied()
13515                .filter(|event| event.event_type == "TimerScheduled")
13516                .collect();
13517            let fired: Vec<_> = timer_events
13518                .iter()
13519                .copied()
13520                .filter(|event| event.event_type == "TimerFired")
13521                .collect();
13522            if scheduled.len() != 1 {
13523                return Err(invalid_recorded_history(
13524                    "timer_schedule_missing_or_duplicate",
13525                    sequence,
13526                    "one TimerScheduled event",
13527                    &format!("{} TimerScheduled events", scheduled.len()),
13528                    "timer replay requires exactly one recorded schedule event",
13529                ));
13530            }
13531            if fired.len() > 1 {
13532                return Err(invalid_recorded_history(
13533                    "duplicate_timer_fire",
13534                    sequence,
13535                    "at most one TimerFired event",
13536                    "multiple TimerFired events",
13537                    "timer history contains more than one fire event for a workflow sequence",
13538                ));
13539            }
13540
13541            let scheduled = scheduled[0];
13542            let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13543            let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13544            if let Some(fired) = fired.first() {
13545                let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13546                if fired_timer_id != timer_id {
13547                    return Err(invalid_recorded_history(
13548                        "timer_identity_mismatch",
13549                        sequence,
13550                        &timer_id,
13551                        &fired_timer_id,
13552                        "TimerFired does not correspond to the recorded TimerScheduled event",
13553                    ));
13554                }
13555                let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13556                if fired_delay != delay_seconds {
13557                    return Err(invalid_recorded_history(
13558                        "timer_history_delay_mismatch",
13559                        sequence,
13560                        &delay_seconds.to_string(),
13561                        &fired_delay.to_string(),
13562                        "TimerScheduled and TimerFired record different delays",
13563                    ));
13564                }
13565            }
13566
13567            Ok(RecordedCommand::Timer {
13568                sequence,
13569                delay_seconds,
13570                fired: !fired.is_empty(),
13571                parallel_group_path: recorded_parallel_group_path(&timer_events, sequence)?,
13572            })
13573        })
13574        .collect::<Result<_>>()?;
13575
13576    let mut marker_sequences = HashMap::new();
13577    for command in &commands {
13578        if let RecordedCommand::VersionMarker {
13579            sequence,
13580            change_id,
13581            ..
13582        } = command
13583        {
13584            if let Some(first_sequence) = marker_sequences.insert(change_id.clone(), *sequence) {
13585                return Err(invalid_recorded_history(
13586                    "duplicate_version_marker",
13587                    *sequence,
13588                    &format!("one marker for change ID {change_id:?}"),
13589                    &format!("markers at sequences {first_sequence} and {sequence}"),
13590                    "workflow history contains duplicate markers for one stable change ID",
13591                ));
13592            }
13593        }
13594    }
13595
13596    Ok(commands)
13597}
13598
13599fn required_version_i32(payload: &Value, field: &str, sequence: u64) -> Result<i32> {
13600    payload
13601        .get(field)
13602        .and_then(Value::as_i64)
13603        .and_then(|value| i32::try_from(value).ok())
13604        .ok_or_else(|| {
13605            invalid_recorded_history(
13606                "version_marker_field_missing",
13607                sequence,
13608                &format!("integer {field}"),
13609                "missing or out-of-range integer",
13610                "version-marker history is missing a required integer field",
13611            )
13612        })
13613}
13614
13615fn durable_event_sequence(event: &HistoryEvent) -> Option<u64> {
13616    event
13617        .payload
13618        .get("sequence")
13619        .or_else(|| event.payload.get("workflow_sequence"))
13620        .or_else(|| event.raw.get("sequence"))
13621        .or_else(|| event.raw.get("workflow_sequence"))
13622        .and_then(value_as_u64)
13623}
13624
13625fn is_internal_timer_event(event: &HistoryEvent) -> bool {
13626    matches!(
13627        event
13628            .payload
13629            .get("timer_kind")
13630            .or_else(|| event.raw.get("timer_kind"))
13631            .and_then(Value::as_str),
13632        Some("condition_timeout" | "signal_timeout")
13633    )
13634}
13635
13636fn is_recorded_condition_wait_event(event: &HistoryEvent) -> bool {
13637    matches!(
13638        event.event_type.as_str(),
13639        "ConditionWaitOpened" | "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13640    )
13641}
13642
13643fn recorded_condition_wait(
13644    sequence: u64,
13645    condition_events: &[&HistoryEvent],
13646    all_events: &[HistoryEvent],
13647) -> Result<RecordedCommand> {
13648    let opened = condition_events
13649        .iter()
13650        .copied()
13651        .filter(|event| event.event_type == "ConditionWaitOpened")
13652        .collect::<Vec<_>>();
13653    if opened.len() != 1 {
13654        return Err(invalid_recorded_history(
13655            "condition_wait_open_missing_or_duplicate",
13656            sequence,
13657            "one ConditionWaitOpened event",
13658            &format!("{} ConditionWaitOpened events", opened.len()),
13659            "condition replay requires exactly one canonical wait-open event",
13660        ));
13661    }
13662    let terminal = condition_events
13663        .iter()
13664        .copied()
13665        .filter(|event| {
13666            matches!(
13667                event.event_type.as_str(),
13668                "ConditionWaitSatisfied" | "ConditionWaitTimedOut"
13669            )
13670        })
13671        .collect::<Vec<_>>();
13672    if terminal.len() > 1 {
13673        return Err(invalid_recorded_history(
13674            "duplicate_condition_wait_terminal_event",
13675            sequence,
13676            "at most one condition terminal event",
13677            "multiple condition terminal events",
13678            "condition history settles one durable wait more than once",
13679        ));
13680    }
13681
13682    let opened = opened[0];
13683    let condition_wait_id = required_condition_wait_id(opened, sequence)?;
13684    let occurrence_id = required_condition_wait_occurrence_id(opened, sequence)?;
13685    for event in condition_events
13686        .iter()
13687        .copied()
13688        .filter(|event| !std::ptr::eq(*event, opened))
13689    {
13690        let event_wait_id = required_condition_wait_id(event, sequence)?;
13691        if event_wait_id != condition_wait_id {
13692            return Err(invalid_recorded_history(
13693                "condition_wait_id_mismatch",
13694                sequence,
13695                &condition_wait_id,
13696                &event_wait_id,
13697                "condition lifecycle events at one sequence disagree on wait identity",
13698            ));
13699        }
13700        let event_occurrence_id = required_condition_wait_occurrence_id(event, sequence)?;
13701        if event_occurrence_id != occurrence_id {
13702            return Err(invalid_recorded_history(
13703                "condition_wait_occurrence_history_mismatch",
13704                sequence,
13705                &occurrence_id,
13706                &event_occurrence_id,
13707                "condition lifecycle events at one sequence disagree on authored occurrence identity",
13708            ));
13709        }
13710    }
13711
13712    let condition_key = optional_non_empty_history_string(opened, "condition_key");
13713    let predicate_identity = opened
13714        .payload
13715        .get("condition_definition_fingerprint")
13716        .and_then(Value::as_str)
13717        .filter(|value| !value.is_empty())
13718        .map(str::to_string)
13719        .ok_or_else(|| {
13720            invalid_recorded_history(
13721                "condition_wait_predicate_fingerprint_missing",
13722                sequence,
13723                "non-empty condition_definition_fingerprint",
13724                &opened.event_type,
13725                "canonical condition history is missing its predicate identity",
13726            )
13727        })?;
13728    let timeout_seconds = optional_history_u64(opened, "timeout_seconds", sequence)?;
13729    for event in condition_events
13730        .iter()
13731        .copied()
13732        .filter(|event| !std::ptr::eq(*event, opened))
13733    {
13734        for (field, opened_value) in [
13735            ("condition_key", condition_key.as_deref()),
13736            (
13737                "condition_definition_fingerprint",
13738                Some(predicate_identity.as_str()),
13739            ),
13740        ] {
13741            if let Some(value) = optional_non_empty_history_string(event, field) {
13742                if opened_value.is_some_and(|opened_value| opened_value != value) {
13743                    return Err(invalid_recorded_history(
13744                        "condition_wait_definition_history_mismatch",
13745                        sequence,
13746                        opened_value.unwrap_or_default(),
13747                        &value,
13748                        "condition lifecycle events disagree on the recorded definition",
13749                    ));
13750                }
13751            }
13752        }
13753        if let Some(event_timeout) = optional_history_u64(event, "timeout_seconds", sequence)? {
13754            if timeout_seconds.is_some_and(|opened_timeout| opened_timeout != event_timeout) {
13755                return Err(invalid_recorded_history(
13756                    "condition_wait_definition_history_mismatch",
13757                    sequence,
13758                    &format!("{}s", timeout_seconds.unwrap_or_default()),
13759                    &format!("{event_timeout}s"),
13760                    "condition lifecycle events disagree on the recorded timeout",
13761                ));
13762            }
13763        }
13764    }
13765
13766    let timeout_timer_events = all_events
13767        .iter()
13768        .filter(|event| {
13769            matches!(
13770                event.event_type.as_str(),
13771                "TimerScheduled" | "TimerCancelled" | "TimerFired"
13772            ) && event.payload.get("timer_kind").and_then(Value::as_str)
13773                == Some("condition_timeout")
13774                && event
13775                    .payload
13776                    .get("condition_wait_id")
13777                    .and_then(Value::as_str)
13778                    == Some(condition_wait_id.as_str())
13779        })
13780        .collect::<Vec<_>>();
13781    let scheduled = timeout_timer_events
13782        .iter()
13783        .copied()
13784        .filter(|event| event.event_type == "TimerScheduled")
13785        .collect::<Vec<_>>();
13786    let fired = timeout_timer_events
13787        .iter()
13788        .copied()
13789        .filter(|event| event.event_type == "TimerFired")
13790        .collect::<Vec<_>>();
13791    if scheduled.len() > 1 || fired.len() > 1 || (!fired.is_empty() && scheduled.len() != 1) {
13792        return Err(invalid_recorded_history(
13793            "condition_wait_timeout_history_invalid",
13794            sequence,
13795            "one timeout schedule and at most one fire",
13796            &format!("{} schedules and {} fires", scheduled.len(), fired.len()),
13797            "condition timeout history has a missing or duplicate lifecycle event",
13798        ));
13799    }
13800    if let Some(scheduled) = scheduled.first() {
13801        let timer_id = required_history_string(scheduled, "timer_id", sequence)?;
13802        let delay_seconds = required_history_u64(scheduled, "delay_seconds", sequence)?;
13803        if timeout_seconds.is_some_and(|timeout| timeout != delay_seconds) {
13804            return Err(invalid_recorded_history(
13805                "condition_wait_timeout_delay_mismatch",
13806                sequence,
13807                &format!("{}s", timeout_seconds.unwrap_or_default()),
13808                &format!("{delay_seconds}s"),
13809                "condition timeout timer differs from the wait definition",
13810            ));
13811        }
13812        if let Some(fired) = fired.first() {
13813            let fired_timer_id = required_history_string(fired, "timer_id", sequence)?;
13814            let fired_delay = required_history_u64(fired, "delay_seconds", sequence)?;
13815            if fired_timer_id != timer_id || fired_delay != delay_seconds {
13816                return Err(invalid_recorded_history(
13817                    "condition_wait_timeout_identity_mismatch",
13818                    sequence,
13819                    &format!("{timer_id}:{delay_seconds}s"),
13820                    &format!("{fired_timer_id}:{fired_delay}s"),
13821                    "condition timeout fire does not match its durable schedule",
13822                ));
13823            }
13824        }
13825    }
13826
13827    let result = terminal.first().map(|event| {
13828        if event.event_type == "ConditionWaitTimedOut" {
13829            ConditionWaitResult::TimedOut
13830        } else {
13831            ConditionWaitResult::Satisfied
13832        }
13833    });
13834    let result = if !fired.is_empty() {
13835        if result == Some(ConditionWaitResult::Satisfied) {
13836            return Err(invalid_recorded_history(
13837                "condition_wait_terminal_conflict",
13838                sequence,
13839                "one satisfied or timed-out outcome",
13840                "satisfied event and fired timeout",
13841                "condition history records conflicting terminal outcomes",
13842            ));
13843        }
13844        Some(ConditionWaitResult::TimedOut)
13845    } else {
13846        result
13847    };
13848
13849    Ok(RecordedCommand::ConditionWait {
13850        sequence,
13851        occurrence_id,
13852        condition_key,
13853        predicate_identity,
13854        timeout_seconds,
13855        result,
13856        parallel_group_path: recorded_parallel_group_path(condition_events, sequence)?,
13857    })
13858}
13859
13860fn required_condition_wait_occurrence_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13861    event
13862        .payload
13863        .get("condition_wait_occurrence_id")
13864        .and_then(Value::as_str)
13865        .filter(|value| !value.is_empty())
13866        .map(str::to_string)
13867        .ok_or_else(|| {
13868            invalid_recorded_history(
13869                "condition_wait_occurrence_id_missing",
13870                sequence,
13871                "non-empty condition_wait_occurrence_id",
13872                &event.event_type,
13873                "condition history is missing authored occurrence identity",
13874            )
13875        })
13876}
13877
13878fn required_condition_wait_id(event: &HistoryEvent, sequence: u64) -> Result<String> {
13879    event
13880        .payload
13881        .get("condition_wait_id")
13882        .and_then(Value::as_str)
13883        .filter(|value| !value.is_empty())
13884        .map(str::to_string)
13885        .ok_or_else(|| {
13886            invalid_recorded_history(
13887                "condition_wait_id_missing",
13888                sequence,
13889                "non-empty condition_wait_id",
13890                &event.event_type,
13891                "canonical condition history is missing its durable wait identity",
13892            )
13893        })
13894}
13895
13896fn optional_non_empty_history_string(event: &HistoryEvent, field: &str) -> Option<String> {
13897    event
13898        .payload
13899        .get(field)
13900        .and_then(Value::as_str)
13901        .filter(|value| !value.is_empty())
13902        .map(str::to_string)
13903}
13904
13905fn optional_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<Option<u64>> {
13906    match event.payload.get(field) {
13907        None | Some(Value::Null) => Ok(None),
13908        Some(value) => value_as_u64(value).map(Some).ok_or_else(|| {
13909            invalid_recorded_history(
13910                "condition_wait_definition_invalid",
13911                sequence,
13912                &format!("non-negative integer {field}"),
13913                &value.to_string(),
13914                "condition history contains an invalid numeric definition field",
13915            )
13916        }),
13917    }
13918}
13919
13920fn required_signal_wait_name(event: &HistoryEvent, sequence: u64) -> Result<String> {
13921    event
13922        .payload
13923        .get("signal_name")
13924        .or_else(|| event.raw.get("signal_name"))
13925        .and_then(Value::as_str)
13926        .filter(|value| !value.is_empty())
13927        .map(str::to_string)
13928        .ok_or_else(|| {
13929            invalid_recorded_history(
13930                "signal_wait_name_missing",
13931                sequence,
13932                "non-empty signal_name",
13933                &event.event_type,
13934                "canonical signal-wait history is missing its signal identity",
13935            )
13936        })
13937}
13938
13939fn is_recorded_signal_wait_event(event: &HistoryEvent) -> bool {
13940    matches!(
13941        event.event_type.as_str(),
13942        "SignalWaitOpened" | "SignalApplied"
13943    )
13944}
13945
13946fn required_history_string(event: &HistoryEvent, field: &str, sequence: u64) -> Result<String> {
13947    event
13948        .payload
13949        .get(field)
13950        .and_then(Value::as_str)
13951        .filter(|value| !value.is_empty())
13952        .map(str::to_string)
13953        .ok_or_else(|| {
13954            invalid_recorded_history(
13955                "timer_history_field_missing",
13956                sequence,
13957                field,
13958                &event.event_type,
13959                "timer history is missing a required identity field",
13960            )
13961        })
13962}
13963
13964fn required_history_u64(event: &HistoryEvent, field: &str, sequence: u64) -> Result<u64> {
13965    event
13966        .payload
13967        .get(field)
13968        .and_then(value_as_u64)
13969        .ok_or_else(|| {
13970            invalid_recorded_history(
13971                "timer_history_field_missing",
13972                sequence,
13973                field,
13974                &event.event_type,
13975                "timer history is missing a required numeric field",
13976            )
13977        })
13978}
13979
13980fn recorded_search_attribute_types(
13981    payload: &Value,
13982    attributes: &Value,
13983    sequence: u64,
13984) -> Result<RecordedSnapshotValue<BTreeMap<String, String>>> {
13985    let Some(raw_types) = payload.get("attribute_types") else {
13986        // This is the explicit compatibility rule for histories recorded
13987        // before typed identity was persisted. Values still constrain replay;
13988        // the unknown type snapshot does not assert a typed match.
13989        return Ok(RecordedSnapshotValue::Unknown);
13990    };
13991    let Some(raw_types) = raw_types.as_object() else {
13992        return Err(invalid_recorded_history(
13993            "search_attribute_types_malformed",
13994            sequence,
13995            "canonical attribute type map",
13996            &raw_types.to_string(),
13997            "search-attribute history contains malformed type identity",
13998        ));
13999    };
14000    let attribute_keys = attributes
14001        .as_object()
14002        .expect("recorded search attributes were validated as an object");
14003    let mut types = BTreeMap::new();
14004    for (key, value) in raw_types {
14005        let Some(attribute_type) = value.as_str() else {
14006            return Err(invalid_recorded_history(
14007                "search_attribute_types_malformed",
14008                sequence,
14009                "canonical string type name",
14010                &value.to_string(),
14011                "search-attribute history contains a non-string type identity",
14012            ));
14013        };
14014        if !attribute_keys.contains_key(key)
14015            || !matches!(
14016                attribute_type,
14017                "string" | "keyword" | "keyword_list" | "int" | "float" | "bool" | "datetime"
14018            )
14019        {
14020            return Err(invalid_recorded_history(
14021                "search_attribute_types_malformed",
14022                sequence,
14023                "canonical types for keys present in attributes",
14024                &format!("{key}:{attribute_type}"),
14025                "search-attribute history contains unsupported or orphaned type identity",
14026            ));
14027        }
14028        types.insert(key.clone(), attribute_type.to_string());
14029    }
14030    Ok(RecordedSnapshotValue::Known(types))
14031}
14032
14033fn invalid_recorded_history(
14034    reason: &str,
14035    sequence: u64,
14036    expected: &str,
14037    actual: &str,
14038    message: &str,
14039) -> Error {
14040    Error::NonDeterministicReplay(ReplayFailure::new(
14041        reason,
14042        Some(sequence),
14043        Some(expected.to_string()),
14044        Some(actual.to_string()),
14045        message,
14046    ))
14047}
14048
14049type ActivityOutcome = std::result::Result<AvroValue, ActivityFailure>;
14050
14051fn activity_outcome(
14052    event: &HistoryEvent,
14053    fallback_codec: &str,
14054    recorded_activity_type: Option<String>,
14055) -> Result<ActivityOutcome> {
14056    if event.event_type == "ActivityCompleted" {
14057        let codec = event
14058            .payload
14059            .get("payload_codec")
14060            .and_then(Value::as_str)
14061            .unwrap_or(fallback_codec);
14062        return Ok(Ok(decode_wire_avro_value(
14063            event.payload.get("result").unwrap_or(&Value::Null),
14064            codec,
14065        )?));
14066    }
14067
14068    let payload = &event.payload;
14069    let (kind, fallback_reason, fallback_message) = match event.event_type.as_str() {
14070        "ActivityFailed" => (ActivityFailureKind::Failed, "activity", "activity failed"),
14071        "ActivityCancelled" => (
14072            ActivityFailureKind::Cancelled,
14073            "cancelled",
14074            "activity was cancelled",
14075        ),
14076        "ActivityTimedOut" => (
14077            ActivityFailureKind::TimedOut,
14078            "timeout",
14079            "activity timed out",
14080        ),
14081        _ => unreachable!("activity_outcome is called only for terminal activity events"),
14082    };
14083    let exception = payload
14084        .get("exception")
14085        .filter(|value| !value.is_null())
14086        .cloned();
14087    let failure_category = payload_string(payload, "failure_category");
14088    let timeout_kind = payload_string(payload, "timeout_kind");
14089    let reason = payload_string(payload, "reason").unwrap_or_else(|| match kind {
14090        ActivityFailureKind::Failed => failure_category
14091            .clone()
14092            .unwrap_or_else(|| fallback_reason.to_string()),
14093        ActivityFailureKind::Cancelled => fallback_reason.to_string(),
14094        ActivityFailureKind::TimedOut => timeout_kind
14095            .clone()
14096            .unwrap_or_else(|| fallback_reason.to_string()),
14097    });
14098    let message = payload_string(payload, "message")
14099        .or_else(|| {
14100            exception
14101                .as_ref()
14102                .and_then(|value| payload_string(value, "message"))
14103        })
14104        .unwrap_or_else(|| fallback_message.to_string());
14105
14106    Ok(Err(ActivityFailure {
14107        kind,
14108        reason,
14109        message,
14110        activity_execution_id: payload_string(payload, "activity_execution_id"),
14111        activity_attempt_id: payload_string(payload, "activity_attempt_id"),
14112        activity_type: payload_string(payload, "activity_type")
14113            .or_else(|| payload_string(payload, "activity_name"))
14114            .or(recorded_activity_type),
14115        activity_class: payload_string(payload, "activity_class"),
14116        attempt_number: payload.get("attempt_number").and_then(value_as_u64),
14117        failure_id: payload_string(payload, "failure_id"),
14118        failure_category,
14119        timeout_kind,
14120        non_retryable: payload
14121            .get("non_retryable")
14122            .and_then(Value::as_bool)
14123            .unwrap_or(false),
14124        exception_type: payload_string(payload, "exception_type").or_else(|| {
14125            exception
14126                .as_ref()
14127                .and_then(|value| payload_string(value, "type"))
14128        }),
14129        exception_class: payload_string(payload, "exception_class").or_else(|| {
14130            exception
14131                .as_ref()
14132                .and_then(|value| payload_string(value, "class"))
14133        }),
14134        code: payload
14135            .get("code")
14136            .filter(|value| !value.is_null())
14137            .cloned(),
14138        exception,
14139    }))
14140}
14141
14142type ChildWorkflowOutcome = std::result::Result<ChildWorkflowAvroResult, ChildWorkflowFailure>;
14143
14144fn child_workflow_outcomes(
14145    events: &[HistoryEvent],
14146    fallback_codec: &str,
14147    parent: WorkflowIdentity,
14148) -> Result<Vec<ChildWorkflowOutcome>> {
14149    let mut outcomes = Vec::new();
14150
14151    for event in events {
14152        let kind = match event.event_type.as_str() {
14153            "ChildRunCompleted" => None,
14154            "ChildRunFailed" => Some((
14155                ChildWorkflowFailureKind::Failed,
14156                "child_workflow",
14157                "child workflow failed",
14158            )),
14159            "ChildRunCancelled" => Some((
14160                ChildWorkflowFailureKind::Cancelled,
14161                "cancelled",
14162                "child workflow was cancelled",
14163            )),
14164            "ChildRunTerminated" => Some((
14165                ChildWorkflowFailureKind::Terminated,
14166                "terminated",
14167                "child workflow was terminated",
14168            )),
14169            _ => continue,
14170        };
14171        let payload = &event.payload;
14172        let child_workflow_id = payload_string(payload, "child_workflow_instance_id");
14173        let child_workflow_run_id = payload_string(payload, "child_workflow_run_id");
14174        let child_workflow_type = payload_string(payload, "child_workflow_type");
14175
14176        if let Some((kind, reason, fallback_message)) = kind {
14177            let exception = payload
14178                .get("exception")
14179                .filter(|value| !value.is_null())
14180                .cloned();
14181            let message = payload_string(payload, "message")
14182                .or_else(|| {
14183                    exception
14184                        .as_ref()
14185                        .and_then(|value| payload_string(value, "message"))
14186                })
14187                .unwrap_or_else(|| fallback_message.to_string());
14188            let exception_type = payload_string(payload, "exception_type").or_else(|| {
14189                exception
14190                    .as_ref()
14191                    .and_then(|value| payload_string(value, "type"))
14192            });
14193            let exception_class = payload_string(payload, "exception_class").or_else(|| {
14194                exception
14195                    .as_ref()
14196                    .and_then(|value| payload_string(value, "class"))
14197            });
14198            outcomes.push(Err(ChildWorkflowFailure {
14199                kind,
14200                reason: reason.to_string(),
14201                message,
14202                parent_workflow_id: parent.workflow_id.clone(),
14203                parent_workflow_run_id: parent.run_id.clone(),
14204                child_workflow_id,
14205                child_workflow_run_id,
14206                child_workflow_type,
14207                failure_id: payload_string(payload, "failure_id"),
14208                failure_category: payload_string(payload, "failure_category"),
14209                exception_type,
14210                exception_class,
14211                non_retryable: payload
14212                    .get("non_retryable")
14213                    .and_then(Value::as_bool)
14214                    .unwrap_or(false),
14215                code: payload
14216                    .get("code")
14217                    .filter(|value| !value.is_null())
14218                    .cloned(),
14219                exception,
14220            }));
14221            continue;
14222        }
14223
14224        let codec = payload
14225            .get("payload_codec")
14226            .and_then(Value::as_str)
14227            .unwrap_or(fallback_codec);
14228        let result = payload
14229            .get("result")
14230            .or_else(|| payload.get("output"))
14231            .unwrap_or(&Value::Null);
14232        outcomes.push(Ok(ChildWorkflowAvroResult {
14233            parent: parent.clone(),
14234            child: WorkflowIdentity {
14235                workflow_id: child_workflow_id,
14236                run_id: child_workflow_run_id,
14237            },
14238            child_workflow_type,
14239            result: decode_wire_avro_value(result, codec)?,
14240        }));
14241    }
14242
14243    Ok(outcomes)
14244}
14245
14246fn payload_string(payload: &Value, key: &str) -> Option<String> {
14247    payload
14248        .get(key)
14249        .and_then(Value::as_str)
14250        .filter(|value| !value.is_empty())
14251        .map(str::to_string)
14252}
14253
14254fn workflow_failure_command(error: &Error) -> Value {
14255    let (exception_type, exception_class, properties) = match error {
14256        Error::ActivityFailed(failure) => (
14257            match failure.kind {
14258                ActivityFailureKind::Failed => "ActivityFailed",
14259                ActivityFailureKind::Cancelled => "ActivityCancelled",
14260                ActivityFailureKind::TimedOut => "ActivityTimedOut",
14261            },
14262            "durable_workflow::ActivityFailure",
14263            json!({
14264                "reason": failure.reason,
14265                "activity_execution_id": failure.activity_execution_id,
14266                "activity_attempt_id": failure.activity_attempt_id,
14267                "activity_type": failure.activity_type,
14268                "activity_class": failure.activity_class,
14269                "attempt_number": failure.attempt_number,
14270                "failure_id": failure.failure_id,
14271                "failure_category": failure.failure_category,
14272                "timeout_kind": failure.timeout_kind,
14273                "activity_non_retryable": failure.non_retryable,
14274                "activity_exception_type": failure.exception_type,
14275                "activity_exception_class": failure.exception_class,
14276                "activity_code": failure.code,
14277                "activity_exception": failure.exception,
14278            }),
14279        ),
14280        Error::ChildWorkflowFailed(failure) => (
14281            match failure.kind {
14282                ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14283                ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14284                ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14285            },
14286            "durable_workflow::ChildWorkflowFailure",
14287            json!({
14288                "reason": failure.reason,
14289                "parent_workflow_id": failure.parent_workflow_id,
14290                "parent_workflow_run_id": failure.parent_workflow_run_id,
14291                "child_workflow_id": failure.child_workflow_id,
14292                "child_workflow_run_id": failure.child_workflow_run_id,
14293                "child_workflow_type": failure.child_workflow_type,
14294                "failure_id": failure.failure_id,
14295                "failure_category": failure.failure_category,
14296                "child_exception_type": failure.exception_type,
14297                "child_exception_class": failure.exception_class,
14298                "child_non_retryable": failure.non_retryable,
14299                "child_code": failure.code,
14300                "child_exception": failure.exception,
14301            }),
14302        ),
14303        Error::ParallelFailed(failure) => (
14304            "ParallelFailed",
14305            "durable_workflow::ParallelFailure",
14306            json!({
14307                "parallel_group_id": failure.group_id,
14308                "parallel_member_path": failure.member_path,
14309                "parallel_group_path": failure.group_path,
14310                "completed_members": failure.completed.iter().map(|completion| &completion.member_path).collect::<Vec<_>>(),
14311                "cause_type": workflow_error_type(&failure.cause),
14312                "cause_message": failure.cause.to_string(),
14313            }),
14314        ),
14315        Error::SagaCompensationFailed(failure) => (
14316            "SagaCompensationFailed",
14317            "durable_workflow::SagaCompensationFailure",
14318            json!({
14319                "initiating_failure_type": workflow_error_type(&failure.initiating_failure),
14320                "initiating_failure_message": failure.initiating_failure.to_string(),
14321                "compensation_activity_type": failure.compensation_activity_type,
14322                "compensation_registration_order": failure.compensation_registration_order,
14323                "compensation_failure_type": workflow_error_type(&failure.compensation_failure),
14324                "compensation_failure_message": failure.compensation_failure.to_string(),
14325            }),
14326        ),
14327        Error::WorkflowCancellationRequested(_) => (
14328            "WorkflowCancellationRequested",
14329            "durable_workflow::WorkflowCancellationRequested",
14330            json!({"reason": "cancelled"}),
14331        ),
14332        Error::NonDeterministicReplay(_) => (
14333            "NonDeterministicReplay",
14334            "durable_workflow::Error",
14335            Value::Null,
14336        ),
14337        _ => ("RustWorkflowError", "durable_workflow::Error", Value::Null),
14338    };
14339    let non_retryable = match error {
14340        Error::ActivityFailed(failure) => failure.non_retryable,
14341        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14342        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14343        Error::SagaCompensationFailed(failure) => {
14344            workflow_error_non_retryable(&failure.compensation_failure)
14345        }
14346        Error::WorkflowCancellationRequested(_) => true,
14347        Error::NonDeterministicReplay(_) => true,
14348        _ => false,
14349    };
14350
14351    json!({
14352        "type": "fail_workflow",
14353        "message": error.to_string(),
14354        "exception_type": exception_type,
14355        "exception_class": exception_class,
14356        "non_retryable": non_retryable,
14357        "exception": {
14358            "type": exception_type,
14359            "class": exception_class,
14360            "message": error.to_string(),
14361            "properties": properties,
14362        }
14363    })
14364}
14365
14366fn workflow_error_type(error: &Error) -> &'static str {
14367    match error {
14368        Error::ActivityFailed(failure) => match failure.kind {
14369            ActivityFailureKind::Failed => "ActivityFailed",
14370            ActivityFailureKind::Cancelled => "ActivityCancelled",
14371            ActivityFailureKind::TimedOut => "ActivityTimedOut",
14372        },
14373        Error::ChildWorkflowFailed(failure) => match failure.kind {
14374            ChildWorkflowFailureKind::Failed => "ChildWorkflowFailed",
14375            ChildWorkflowFailureKind::Cancelled => "ChildWorkflowCancelled",
14376            ChildWorkflowFailureKind::Terminated => "ChildWorkflowTerminated",
14377        },
14378        Error::ParallelFailed(_) => "ParallelFailed",
14379        Error::SagaCompensationFailed(_) => "SagaCompensationFailed",
14380        Error::WorkflowCancellationRequested(_) => "WorkflowCancellationRequested",
14381        Error::NonDeterministicReplay(_) => "NonDeterministicReplay",
14382        _ => "RustWorkflowError",
14383    }
14384}
14385
14386fn workflow_error_non_retryable(error: &Error) -> bool {
14387    match error {
14388        Error::ActivityFailed(failure) => failure.non_retryable,
14389        Error::ChildWorkflowFailed(failure) => failure.non_retryable,
14390        Error::ParallelFailed(failure) => workflow_error_non_retryable(&failure.cause),
14391        Error::SagaCompensationFailed(failure) => {
14392            workflow_error_non_retryable(&failure.compensation_failure)
14393        }
14394        Error::WorkflowCancellationRequested(_) | Error::NonDeterministicReplay(_) => true,
14395        _ => false,
14396    }
14397}
14398
14399fn workflow_task_integrity_error(error: &Error) -> bool {
14400    matches!(
14401        error,
14402        Error::NonDeterministicReplay(_)
14403            | Error::Protocol(_)
14404            | Error::MissingWorkflowCommandIdentity
14405            | Error::WorkflowStatePoisoned
14406    )
14407}
14408
14409fn decode_signal_event_arguments(
14410    event: &HistoryEvent,
14411    fallback_codec: &str,
14412) -> Result<Vec<AvroValue>> {
14413    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14414    validate_payload_codec(codec)?;
14415    let raw = signal_history_payload(&event.payload);
14416    let decoded = match raw.filter(|value| !value.is_null()) {
14417        Some(value) => decode_wire_avro_value(value, codec)?,
14418        None => AvroValue::Array(Vec::new()),
14419    };
14420    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14421        unreachable!("normalize_avro_arguments always returns an array");
14422    };
14423    Ok(arguments)
14424}
14425
14426fn decode_update_event_arguments(
14427    event: &HistoryEvent,
14428    fallback_codec: &str,
14429) -> Result<Vec<AvroValue>> {
14430    let codec = declared_payload_codec(&event.payload, "payload_codec")?.unwrap_or(fallback_codec);
14431    validate_payload_codec(codec)?;
14432    let decoded = match event
14433        .payload
14434        .get("arguments")
14435        .filter(|value| !value.is_null())
14436    {
14437        Some(value) => decode_wire_avro_value(value, codec)?,
14438        None => AvroValue::Array(Vec::new()),
14439    };
14440    let AvroValue::Array(arguments) = normalize_avro_arguments(decoded) else {
14441        unreachable!("normalize_avro_arguments always returns an array");
14442    };
14443    Ok(arguments)
14444}
14445
14446fn hydrate_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14447    let Some(export_events) = task
14448        .history_export
14449        .as_ref()
14450        .and_then(|export| export.get("history_events"))
14451        .and_then(Value::as_array)
14452    else {
14453        return Ok(());
14454    };
14455
14456    if export_events.len() > task.history_events.len() {
14457        task.history_events = serde_json::from_value(Value::Array(export_events.clone()))?;
14458    }
14459
14460    Ok(())
14461}
14462
14463fn enrich_query_history_from_export(task: &mut QueryTask) -> Result<()> {
14464    let Some(export) = task.history_export.as_ref() else {
14465        return Ok(());
14466    };
14467    let signals = export
14468        .get("signals")
14469        .and_then(Value::as_array)
14470        .cloned()
14471        .unwrap_or_default();
14472    let activities = export
14473        .get("activities")
14474        .and_then(Value::as_array)
14475        .cloned()
14476        .unwrap_or_default();
14477    let export_codec = export
14478        .get("payloads")
14479        .and_then(|payloads| payloads.get("codec"))
14480        .and_then(Value::as_str)
14481        .unwrap_or(&task.payload_codec)
14482        .to_string();
14483    let mut signal_name_offsets: HashMap<String, usize> = HashMap::new();
14484
14485    for event in &mut task.history_events {
14486        if event.event_type == "ActivityCompleted" {
14487            let sequence = event
14488                .payload
14489                .get("sequence")
14490                .or_else(|| event.payload.get("workflow_sequence"))
14491                .and_then(value_as_u64);
14492            let Some(activity) = sequence.and_then(|sequence| {
14493                activities.iter().find(|activity| {
14494                    activity.get("sequence").and_then(value_as_u64) == Some(sequence)
14495                })
14496            }) else {
14497                continue;
14498            };
14499            let Some(payload) = event.payload.as_object_mut() else {
14500                continue;
14501            };
14502            if missing_payload(payload.get("result")) {
14503                if let Some(result) = activity
14504                    .get("result")
14505                    .filter(|value| !missing_payload(Some(value)))
14506                {
14507                    payload.insert("result".to_string(), result.clone());
14508                }
14509            }
14510            for field in ["payload_codec", "activity_type"] {
14511                if payload
14512                    .get(field)
14513                    .and_then(Value::as_str)
14514                    .unwrap_or_default()
14515                    .is_empty()
14516                {
14517                    if let Some(value) = activity.get(field) {
14518                        payload.insert(field.to_string(), value.clone());
14519                    }
14520                }
14521            }
14522            continue;
14523        }
14524
14525        if event.event_type != "SignalReceived" && event.event_type != "SignalApplied" {
14526            continue;
14527        }
14528        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14529        let command_id = event
14530            .payload
14531            .get("workflow_command_id")
14532            .or_else(|| event.raw.get("workflow_command_id"))
14533            .and_then(Value::as_str);
14534        let signal_name = event
14535            .payload
14536            .get("signal_name")
14537            .and_then(Value::as_str)
14538            .unwrap_or_default()
14539            .to_string();
14540        let matched = signals
14541            .iter()
14542            .find(|signal| {
14543                signal_id.is_some() && signal.get("id").and_then(Value::as_str) == signal_id
14544            })
14545            .or_else(|| {
14546                signals.iter().find(|signal| {
14547                    command_id.is_some()
14548                        && signal.get("command_id").and_then(Value::as_str) == command_id
14549                })
14550            })
14551            .or_else(|| {
14552                let offset = signal_name_offsets.entry(signal_name.clone()).or_default();
14553                let signal = signals
14554                    .iter()
14555                    .filter(|signal| {
14556                        signal.get("name").and_then(Value::as_str) == Some(signal_name.as_str())
14557                    })
14558                    .nth(*offset);
14559                if signal.is_some() {
14560                    *offset += 1;
14561                }
14562                signal
14563            });
14564        let Some(signal) = matched else {
14565            continue;
14566        };
14567        let signal_codec = signal
14568            .get("payload_codec")
14569            .and_then(Value::as_str)
14570            .unwrap_or(&export_codec);
14571        let Some(payload) = event.payload.as_object_mut() else {
14572            continue;
14573        };
14574        if missing_payload(payload.get("arguments")) {
14575            if let Some(arguments) = signal
14576                .get("arguments")
14577                .filter(|value| !missing_payload(Some(value)))
14578            {
14579                let envelope = match arguments {
14580                    Value::String(blob) => json!({"codec": signal_codec, "blob": blob}),
14581                    other => other.clone(),
14582                };
14583                payload.insert("arguments".to_string(), envelope);
14584            }
14585        }
14586        if payload
14587            .get("payload_codec")
14588            .and_then(Value::as_str)
14589            .unwrap_or_default()
14590            .is_empty()
14591        {
14592            payload.insert("payload_codec".to_string(), json!(signal_codec));
14593        }
14594    }
14595
14596    Ok(())
14597}
14598
14599fn missing_payload(value: Option<&Value>) -> bool {
14600    match value {
14601        None | Some(Value::Null) => true,
14602        Some(Value::String(value)) => value.is_empty(),
14603        Some(_) => false,
14604    }
14605}
14606
14607fn query_signal_events(task: &QueryTask) -> Result<Vec<QuerySignal>> {
14608    let export_signals = task
14609        .history_export
14610        .as_ref()
14611        .and_then(|export| export.get("signals"))
14612        .and_then(Value::as_array)
14613        .cloned()
14614        .unwrap_or_default();
14615    let export_codec = task
14616        .history_export
14617        .as_ref()
14618        .and_then(|export| export.get("payloads"))
14619        .and_then(|payloads| payloads.get("codec"))
14620        .and_then(Value::as_str)
14621        .unwrap_or(&task.payload_codec);
14622    let mut name_offsets: HashMap<String, usize> = HashMap::new();
14623    let mut signals = Vec::new();
14624
14625    for event in &task.history_events {
14626        if event.event_type != "SignalApplied" && event.event_type != "SignalReceived" {
14627            continue;
14628        }
14629
14630        let name = event
14631            .payload
14632            .get("signal_name")
14633            .and_then(Value::as_str)
14634            .unwrap_or_default();
14635        if name.is_empty() {
14636            continue;
14637        }
14638        let signal_id = event.payload.get("signal_id").and_then(Value::as_str);
14639        let command_id = event
14640            .payload
14641            .get("workflow_command_id")
14642            .or_else(|| event.raw.get("workflow_command_id"))
14643            .and_then(Value::as_str);
14644        let matched_export = export_signals
14645            .iter()
14646            .find(|candidate| {
14647                signal_id.is_some() && candidate.get("id").and_then(Value::as_str) == signal_id
14648            })
14649            .or_else(|| {
14650                export_signals.iter().find(|candidate| {
14651                    command_id.is_some()
14652                        && candidate.get("command_id").and_then(Value::as_str) == command_id
14653                })
14654            })
14655            .or_else(|| {
14656                let offset = name_offsets.entry(name.to_string()).or_default();
14657                let candidate = export_signals
14658                    .iter()
14659                    .filter(|candidate| candidate.get("name").and_then(Value::as_str) == Some(name))
14660                    .nth(*offset);
14661                if candidate.is_some() {
14662                    *offset += 1;
14663                }
14664                candidate
14665            });
14666        let codec = event
14667            .payload
14668            .get("payload_codec")
14669            .and_then(Value::as_str)
14670            .or_else(|| {
14671                matched_export
14672                    .and_then(|signal| signal.get("payload_codec"))
14673                    .and_then(Value::as_str)
14674            })
14675            .unwrap_or(export_codec);
14676        let raw_arguments = signal_history_payload(&event.payload)
14677            .filter(|value| !value.is_null())
14678            .or_else(|| matched_export.and_then(|signal| signal.get("arguments")));
14679        let (arguments, avro_arguments) = decode_query_signal_arguments(raw_arguments, codec)?;
14680        let workflow_sequence = event
14681            .payload
14682            .get("workflow_sequence")
14683            .and_then(value_as_u64)
14684            .or_else(|| {
14685                matched_export
14686                    .and_then(|signal| signal.get("workflow_sequence"))
14687                    .and_then(value_as_u64)
14688            });
14689
14690        signals.push(QuerySignal {
14691            id: signal_id.map(str::to_string).or_else(|| {
14692                matched_export
14693                    .and_then(|signal| signal.get("id"))
14694                    .and_then(Value::as_str)
14695                    .map(str::to_string)
14696            }),
14697            name: name.to_string(),
14698            arguments,
14699            avro_arguments,
14700            workflow_sequence,
14701        });
14702    }
14703
14704    if signals.is_empty() {
14705        for signal in export_signals {
14706            if signal.get("status").and_then(Value::as_str) == Some("rejected") {
14707                continue;
14708            }
14709            let Some(name) = signal.get("name").and_then(Value::as_str) else {
14710                continue;
14711            };
14712            let codec = signal
14713                .get("payload_codec")
14714                .and_then(Value::as_str)
14715                .unwrap_or(export_codec);
14716            let (arguments, avro_arguments) =
14717                decode_query_signal_arguments(signal.get("arguments"), codec)?;
14718            signals.push(QuerySignal {
14719                id: signal.get("id").and_then(Value::as_str).map(str::to_string),
14720                name: name.to_string(),
14721                arguments,
14722                avro_arguments,
14723                workflow_sequence: signal.get("workflow_sequence").and_then(value_as_u64),
14724            });
14725        }
14726        signals.sort_by_key(|signal| signal.workflow_sequence.unwrap_or(u64::MAX));
14727    }
14728
14729    Ok(signals)
14730}
14731
14732fn decode_query_signal_arguments(
14733    raw: Option<&Value>,
14734    codec: &str,
14735) -> Result<(Vec<Value>, Vec<AvroValue>)> {
14736    validate_payload_codec(codec)?;
14737    let decoded = match raw.filter(|value| !value.is_null()) {
14738        Some(value) => decode_wire_avro_value(value, codec)?,
14739        None => AvroValue::Array(Vec::new()),
14740    };
14741    let AvroValue::Array(avro_arguments) = normalize_avro_arguments(decoded) else {
14742        unreachable!("normalize_avro_arguments always returns an array");
14743    };
14744    let arguments = avro_arguments
14745        .iter()
14746        .cloned()
14747        .map(AvroValue::into_json)
14748        .collect::<Result<Vec<_>>>()?;
14749    Ok((arguments, avro_arguments))
14750}
14751
14752fn value_as_u64(value: &Value) -> Option<u64> {
14753    value
14754        .as_u64()
14755        .or_else(|| value.as_str().and_then(|value| value.parse().ok()))
14756}
14757
14758#[cfg(test)]
14759mod tests {
14760    use super::*;
14761    mod runtime_payloads;
14762    mod runtime_uploads;
14763    use std::{
14764        fs,
14765        io::{Read, Write},
14766        net::{SocketAddr, TcpListener, TcpStream},
14767        process::Command as ProcessCommand,
14768        sync::atomic::AtomicUsize,
14769        thread,
14770    };
14771
14772    #[derive(Clone, Copy, Debug)]
14773    enum InvalidTaskPayloadCodec {
14774        Missing,
14775        Null,
14776        NonString,
14777    }
14778
14779    impl InvalidTaskPayloadCodec {
14780        fn label(self) -> &'static str {
14781            match self {
14782                Self::Missing => "missing",
14783                Self::Null => "null",
14784                Self::NonString => "non-string",
14785            }
14786        }
14787
14788        fn apply(self, task: &mut Value) {
14789            let task = task.as_object_mut().expect("task fixture object");
14790            match self {
14791                Self::Missing => {
14792                    task.remove("payload_codec");
14793                }
14794                Self::Null => {
14795                    task.insert("payload_codec".to_string(), Value::Null);
14796                }
14797                Self::NonString => {
14798                    task.insert("payload_codec".to_string(), json!(42));
14799                }
14800            }
14801        }
14802    }
14803
14804    fn fixture_envelope(value: Value) -> Value {
14805        encode_value_envelope(&value, DEFAULT_CODEC).expect("encode Avro test fixture")
14806    }
14807
14808    fn fixture_blob(value: Value) -> String {
14809        encode_payload(&value, DEFAULT_CODEC)
14810            .expect("encode Avro test fixture")
14811            .blob
14812    }
14813
14814    #[test]
14815    fn client_builder_rejects_the_sdk_owned_api_suffix() {
14816        for base_url in [
14817            "http://127.0.0.1:8080/api",
14818            "http://localhost:8080/api/",
14819            "https://runtime.example.test/namespaces/orders/api",
14820        ] {
14821            let error = Client::builder(base_url)
14822                .build()
14823                .expect_err("SDK-owned /api suffix must be rejected during build");
14824
14825            assert!(matches!(error, Error::InvalidBaseUrl), "{base_url}");
14826            assert!(
14827                error.to_string().contains("SDK appends /api automatically"),
14828                "the validation error must explain how to fix the endpoint"
14829            );
14830        }
14831    }
14832
14833    #[test]
14834    fn client_builder_preserves_self_hosted_and_managed_runtime_prefixes() {
14835        for (base_url, expected) in [
14836            ("http://127.0.0.1:8080", "http://127.0.0.1:8080"),
14837            (
14838                "http://localhost:8080/durable-workflow/",
14839                "http://localhost:8080/durable-workflow",
14840            ),
14841            (
14842                "https://runtime.example.test/namespaces/orders",
14843                "https://runtime.example.test/namespaces/orders",
14844            ),
14845            (
14846                "https://runtime.example.test/gateway/api/namespaces/orders",
14847                "https://runtime.example.test/gateway/api/namespaces/orders",
14848            ),
14849            (
14850                "https://api.example.test/runtime/orders/",
14851                "https://api.example.test/runtime/orders",
14852            ),
14853        ] {
14854            let client = Client::builder(base_url)
14855                .build()
14856                .expect("Server and Cloud runtime base URL must remain valid");
14857
14858            assert_eq!(client.base_url, expected);
14859        }
14860    }
14861
14862    #[test]
14863    fn workflow_completion_uses_the_additive_command_protocol_floor() {
14864        assert_eq!(
14865            workflow_completion_protocol_version(&[json!({"type": "complete_workflow"})]),
14866            WORKER_PROTOCOL_VERSION
14867        );
14868        assert_eq!(
14869            workflow_completion_protocol_version(&[json!({
14870                "type": "upsert_search_attributes",
14871                "attributes": {"OrderStatus": "waiting"},
14872            })]),
14873            SEARCH_ATTRIBUTE_UPDATE_MINIMUM_WORKER_PROTOCOL_VERSION
14874        );
14875        assert_eq!(
14876            workflow_completion_protocol_version(&[json!({
14877                "type": "upsert_search_attributes",
14878                "attributes": {"OrderStatus": "waiting"},
14879                "attribute_types": {"OrderStatus": "keyword"},
14880            })]),
14881            TYPED_SEARCH_ATTRIBUTES_MINIMUM_WORKER_PROTOCOL_VERSION
14882        );
14883        assert_eq!(
14884            workflow_completion_protocol_version(&[
14885                json!({"type": "upsert_memo", "entries": {"status": "waiting"}}),
14886                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14887            ]),
14888            MEMO_UPSERT_MINIMUM_WORKER_PROTOCOL_VERSION
14889        );
14890        assert_eq!(
14891            workflow_completion_protocol_version(&[
14892                json!({"type": "upsert_search_attributes", "attributes": {"State": "waiting"}}),
14893                json!({"type": "open_condition_wait", "condition_key": "ready"}),
14894            ]),
14895            CONDITION_WAIT_MINIMUM_WORKER_PROTOCOL_VERSION
14896        );
14897        assert_eq!(
14898            workflow_completion_protocol_version(&[json!({
14899                "type": "open_condition_wait",
14900                "condition_wait_occurrence_id": "rust:condition-wait:0",
14901                "condition_key": "ready",
14902            })]),
14903            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14904        );
14905        assert_eq!(
14906            workflow_completion_protocol_version_with_message_streams(
14907                &[json!({"type": "upsert_memo", "entries": {"status": "waiting"}})],
14908                true,
14909            ),
14910            MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION
14911        );
14912        assert_eq!(
14913            workflow_completion_protocol_version_with_message_streams(
14914                &[json!({
14915                    "type": "open_condition_wait",
14916                    "condition_wait_occurrence_id": "rust:condition-wait:0",
14917                    "condition_key": "ready",
14918                })],
14919                true,
14920            ),
14921            CONDITION_WAIT_OCCURRENCE_IDENTITY_MINIMUM_WORKER_PROTOCOL_VERSION
14922        );
14923    }
14924
14925    #[test]
14926    fn portable_worker_affinity_manifest_explicitly_refuses_unimplemented_features() {
14927        let manifest = portable_worker_affinity_capability_manifest();
14928
14929        for capability in ["local_activities", "worker_sessions", "sticky_execution"] {
14930            assert_eq!(manifest[capability]["supported"], json!(false));
14931            assert_eq!(
14932                manifest[capability]["minimum_protocol_version"],
14933                json!(PORTABLE_WORKER_AFFINITY_MINIMUM_PROTOCOL_VERSION)
14934            );
14935            assert!(manifest[capability]["reason"]
14936                .as_str()
14937                .is_some_and(|reason| !reason.is_empty()));
14938        }
14939    }
14940
14941    fn typed_fidelity_probe() -> AvroValue {
14942        AvroValue::Map(BTreeMap::from([
14943            ("bytes".to_string(), AvroValue::Bytes(vec![0, 0xff])),
14944            ("empty".to_string(), AvroValue::Map(BTreeMap::new())),
14945            (
14946                "numeric".to_string(),
14947                AvroValue::Map(BTreeMap::from([
14948                    ("0".to_string(), AvroValue::String("zero".to_string())),
14949                    ("1".to_string(), AvroValue::String("one".to_string())),
14950                ])),
14951            ),
14952            (
14953                "nested".to_string(),
14954                AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([(
14955                    "enabled".to_string(),
14956                    AvroValue::Boolean(true),
14957                )]))]),
14958            ),
14959            (
14960                "projection_collisions".to_string(),
14961                AvroValue::Array(projection_collision_probe()),
14962            ),
14963        ]))
14964    }
14965
14966    fn projection_collision_probe() -> Vec<AvroValue> {
14967        vec![
14968            AvroValue::Map(BTreeMap::from([
14969                ("$type".to_string(), AvroValue::String("bytes".to_string())),
14970                (
14971                    "base64".to_string(),
14972                    AvroValue::String("ordinary user text".to_string()),
14973                ),
14974            ])),
14975            AvroValue::Map(BTreeMap::from([
14976                ("$type".to_string(), AvroValue::String("map".to_string())),
14977                (
14978                    "entries".to_string(),
14979                    AvroValue::Array(vec![AvroValue::Map(BTreeMap::from([
14980                        ("key".to_string(), AvroValue::String("ordinary".to_string())),
14981                        (
14982                            "value".to_string(),
14983                            AvroValue::String("user map".to_string()),
14984                        ),
14985                    ]))]),
14986                ),
14987            ])),
14988        ]
14989    }
14990
14991    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
14992    struct TypedContract {
14993        nested: TypedNested,
14994        mode: TypedMode,
14995        optional: Option<String>,
14996        absent: Option<String>,
14997        items: Vec<i64>,
14998        labels: BTreeMap<String, String>,
14999        bytes: serde_bytes::ByteBuf,
15000        signed: i64,
15001        finite: f64,
15002    }
15003
15004    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15005    struct TypedNested {
15006        enabled: bool,
15007    }
15008
15009    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
15010    enum TypedMode {
15011        Detailed { label: String },
15012    }
15013
15014    fn typed_contract() -> TypedContract {
15015        TypedContract {
15016            nested: TypedNested { enabled: true },
15017            mode: TypedMode::Detailed {
15018                label: "compiler-checked".to_string(),
15019            },
15020            optional: Some("present".to_string()),
15021            absent: None,
15022            items: vec![i64::MIN, 0, i64::MAX],
15023            labels: BTreeMap::from([
15024                ("language".to_string(), "rust".to_string()),
15025                ("wire".to_string(), "avro".to_string()),
15026            ]),
15027            bytes: serde_bytes::ByteBuf::from(vec![0, 0xff, 7]),
15028            signed: -9_223_372_036_854_775_000,
15029            finite: 12.5,
15030        }
15031    }
15032
15033    #[derive(Clone, Debug, Default, PartialEq)]
15034    struct ReplayCounterState {
15035        loaded: Option<String>,
15036        count: i64,
15037        finished: bool,
15038    }
15039
15040    fn replay_counter_worker() -> Worker {
15041        let client = Client::new("http://127.0.0.1:8080").expect("client");
15042        let mut worker = Worker::new(client, "rust-workers");
15043        worker.register_replayed_workflow(
15044            "replay-counter",
15045            ReplayCounterState::default,
15046            |ctx, _input, state| async move {
15047                let loaded = ctx.activity("load-counter", json!([])).await?;
15048                state.update(|current| {
15049                    current.loaded = loaded.as_str().map(str::to_string);
15050                })?;
15051                for _ in 0..2 {
15052                    let signal = ctx.wait_signal("increment").await?;
15053                    let amount = signal.first().and_then(Value::as_i64).unwrap_or_default();
15054                    state.update(|current| current.count += amount)?;
15055                }
15056                state.update(|current| current.finished = true)?;
15057                state.read(|current| Ok(json!(current.count)))?
15058            },
15059        );
15060        worker.register_replayed_query::<ReplayCounterState, _, _>(
15061            "replay-counter",
15062            "current",
15063            |_ctx, state, _args| async move {
15064                Ok(json!({
15065                    "loaded": state.loaded,
15066                    "count": state.count,
15067                    "finished": state.finished,
15068                }))
15069            },
15070        );
15071        worker.register_replayed_query::<ReplayCounterState, _, _>(
15072            "replay-counter",
15073            "detached-mutation",
15074            |_ctx, state, _args| async move {
15075                let mut detached = (*state).clone();
15076                detached.count = 999;
15077                Ok(json!(detached.count))
15078            },
15079        );
15080        worker.register_replayed_query::<ReplayCounterState, _, _>(
15081            "replay-counter",
15082            "failed-mutation",
15083            |_ctx, state, _args| async move {
15084                let mut detached = (*state).clone();
15085                detached.count = 999;
15086                Err(Error::WorkerLoop("query refused".to_string()))
15087            },
15088        );
15089        worker
15090    }
15091
15092    fn replay_counter_query(
15093        query_name: &str,
15094        history_events: Value,
15095        run_status: &str,
15096    ) -> QueryTask {
15097        let arguments = fixture_envelope(json!([]));
15098        serde_json::from_value(json!({
15099            "query_task_id": format!("query-{query_name}"),
15100            "workflow_type": "replay-counter",
15101            "query_name": query_name,
15102            "payload_codec": DEFAULT_CODEC,
15103            "workflow_arguments": arguments.clone(),
15104            "query_arguments": arguments,
15105            "history_events": history_events,
15106            "run_status": run_status,
15107        }))
15108        .expect("query task")
15109    }
15110
15111    fn workflow_context(history: Vec<HistoryEvent>) -> WorkflowContext {
15112        workflow_context_with_codec(history, DEFAULT_CODEC)
15113    }
15114
15115    fn workflow_context_with_codec(
15116        history: Vec<HistoryEvent>,
15117        payload_codec: &str,
15118    ) -> WorkflowContext {
15119        WorkflowContext {
15120            state: Arc::new(Mutex::new(
15121                WorkflowState::new_with_identity(
15122                    history,
15123                    None,
15124                    None,
15125                    "rust-workers".to_string(),
15126                    payload_codec.to_string(),
15127                    None,
15128                )
15129                .expect("valid workflow history"),
15130            )),
15131        }
15132    }
15133
15134    fn history_event(event_type: &str, payload: Value) -> HistoryEvent {
15135        HistoryEvent {
15136            event_type: event_type.to_string(),
15137            payload,
15138            raw: HashMap::new(),
15139        }
15140    }
15141
15142    fn parallel_path_entry(
15143        kind: &str,
15144        base: u64,
15145        size: usize,
15146        index: usize,
15147    ) -> ParallelGroupMetadata {
15148        parallel_group_entry(base, size, index, kind)
15149    }
15150
15151    fn parallel_history_event(
15152        event_type: &str,
15153        sequence: u64,
15154        identity_field: &str,
15155        identity: &str,
15156        path: Vec<ParallelGroupMetadata>,
15157        result: Option<Value>,
15158    ) -> HistoryEvent {
15159        let mut payload = serde_json::Map::from_iter([
15160            ("sequence".to_string(), json!(sequence)),
15161            (identity_field.to_string(), json!(identity)),
15162        ]);
15163        let inner = path.last().expect("parallel history path");
15164        apply_parallel_group_path(&mut payload, std::slice::from_ref(inner));
15165        payload.insert("parallel_group_path".to_string(), json!(path));
15166        if let Some(result) = result {
15167            let field = if event_type == "ChildRunCompleted" {
15168                "result"
15169            } else {
15170                "result"
15171            };
15172            payload.insert(field.to_string(), fixture_envelope(result));
15173            payload.insert("payload_codec".to_string(), json!(DEFAULT_CODEC));
15174        }
15175        history_event(event_type, Value::Object(payload))
15176    }
15177
15178    fn nested_parallel_operations() -> Vec<ParallelOperation> {
15179        vec![
15180            ParallelOperation::activity("first", json!([])),
15181            ParallelOperation::group(vec![
15182                ParallelOperation::child_workflow(
15183                    "second",
15184                    ChildWorkflowOptions::new("child-workers"),
15185                    json!([]),
15186                ),
15187                ParallelOperation::activity("third", json!([])),
15188            ]),
15189        ]
15190    }
15191
15192    fn nested_parallel_paths() -> [Vec<ParallelGroupMetadata>; 3] {
15193        let outer = [
15194            parallel_path_entry("mixed", 1, 3, 0),
15195            parallel_path_entry("mixed", 1, 3, 1),
15196            parallel_path_entry("mixed", 1, 3, 2),
15197        ];
15198        [
15199            vec![outer[0].clone()],
15200            vec![outer[1].clone(), parallel_path_entry("mixed", 2, 2, 0)],
15201            vec![outer[2].clone(), parallel_path_entry("mixed", 2, 2, 1)],
15202        ]
15203    }
15204
15205    #[test]
15206    fn parallel_schedules_every_nested_mixed_leaf_with_stable_metadata() {
15207        let ctx = workflow_context(Vec::new());
15208        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15209        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15210
15211        assert!(matches!(
15212            call.as_mut().poll(&mut task_context),
15213            Poll::Pending
15214        ));
15215        let commands = ctx.take_commands().expect("parallel commands");
15216        assert_eq!(
15217            commands
15218                .iter()
15219                .map(|command| command["type"].as_str().unwrap_or_default())
15220                .collect::<Vec<_>>(),
15221            [
15222                "schedule_activity",
15223                "start_child_workflow",
15224                "schedule_activity"
15225            ]
15226        );
15227        let paths = nested_parallel_paths();
15228        for (command, path) in commands.iter().zip(paths) {
15229            assert_eq!(command["parallel_group_path"], json!(path));
15230            assert_eq!(
15231                command["parallel_group_id"],
15232                json!(path.last().expect("inner group").parallel_group_id)
15233            );
15234        }
15235    }
15236
15237    fn completed_nested_parallel_history() -> Vec<HistoryEvent> {
15238        let paths = nested_parallel_paths();
15239        let third = parallel_history_event(
15240            "ActivityCompleted",
15241            3,
15242            "activity_type",
15243            "third",
15244            paths[2].clone(),
15245            Some(json!("three")),
15246        );
15247        vec![
15248            parallel_history_event(
15249                "ActivityCompleted",
15250                1,
15251                "activity_type",
15252                "first",
15253                paths[0].clone(),
15254                Some(json!("one")),
15255            ),
15256            parallel_history_event(
15257                "ChildWorkflowScheduled",
15258                2,
15259                "child_workflow_type",
15260                "second",
15261                paths[1].clone(),
15262                None,
15263            ),
15264            parallel_history_event(
15265                "ChildRunCompleted",
15266                2,
15267                "child_workflow_type",
15268                "second",
15269                paths[1].clone(),
15270                Some(json!("two")),
15271            ),
15272            third.clone(),
15273            third,
15274        ]
15275    }
15276
15277    #[test]
15278    fn parallel_replay_rebuilds_input_order_and_tolerates_duplicate_delivery() {
15279        for _restart_or_completed_replay in 0..2 {
15280            let ctx = workflow_context(completed_nested_parallel_history());
15281            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15282            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15283            let Poll::Ready(Ok(results)) = call.as_mut().poll(&mut task_context) else {
15284                panic!("completed nested parallel history must replay");
15285            };
15286            assert_eq!(
15287                results,
15288                vec![
15289                    ParallelResult::Activity(json!("one")),
15290                    ParallelResult::Group(vec![
15291                        ParallelResult::ChildWorkflow(ChildWorkflowResult {
15292                            parent: WorkflowIdentity {
15293                                workflow_id: None,
15294                                run_id: None,
15295                            },
15296                            child: WorkflowIdentity {
15297                                workflow_id: None,
15298                                run_id: None,
15299                            },
15300                            child_workflow_type: Some("second".to_string()),
15301                            result: json!("two"),
15302                        }),
15303                        ParallelResult::Activity(json!("three")),
15304                    ]),
15305                ]
15306            );
15307            assert!(ctx.take_commands().expect("commands").is_empty());
15308            ctx.ensure_history_consumed().expect("history consumed");
15309        }
15310    }
15311
15312    #[test]
15313    fn parallel_failure_keeps_typed_cause_path_and_late_completions() {
15314        let paths = nested_parallel_paths();
15315        let history = vec![
15316            parallel_history_event(
15317                "ActivityCompleted",
15318                1,
15319                "activity_type",
15320                "first",
15321                paths[0].clone(),
15322                Some(json!("one")),
15323            ),
15324            parallel_history_event(
15325                "ChildWorkflowScheduled",
15326                2,
15327                "child_workflow_type",
15328                "second",
15329                paths[1].clone(),
15330                None,
15331            ),
15332            parallel_history_event(
15333                "ChildRunFailed",
15334                2,
15335                "child_workflow_type",
15336                "second",
15337                paths[1].clone(),
15338                None,
15339            ),
15340            parallel_history_event(
15341                "ActivityCompleted",
15342                3,
15343                "activity_type",
15344                "third",
15345                paths[2].clone(),
15346                Some(json!("late")),
15347            ),
15348        ];
15349        let ctx = workflow_context(history);
15350        let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15351        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15352        let outcome = call.as_mut().poll(&mut task_context);
15353        let Poll::Ready(Err(Error::ParallelFailed(failure))) = outcome else {
15354            panic!("one failed child must return a typed partial failure: {outcome:?}");
15355        };
15356        assert_eq!(failure.member_path, [1, 0]);
15357        assert_eq!(failure.group_id, "parallel-calls:1:3");
15358        assert!(matches!(*failure.cause, Error::ChildWorkflowFailed(_)));
15359        assert_eq!(
15360            failure
15361                .completed
15362                .iter()
15363                .map(|completion| completion.member_path.clone())
15364                .collect::<Vec<_>>(),
15365            [vec![0], vec![1, 1]]
15366        );
15367    }
15368
15369    #[test]
15370    fn pending_parallel_history_restarts_without_rescheduling_any_leaf() {
15371        let paths = nested_parallel_paths();
15372        let history = vec![
15373            parallel_history_event(
15374                "ActivityScheduled",
15375                1,
15376                "activity_type",
15377                "first",
15378                paths[0].clone(),
15379                None,
15380            ),
15381            parallel_history_event(
15382                "ChildWorkflowScheduled",
15383                2,
15384                "child_workflow_type",
15385                "second",
15386                paths[1].clone(),
15387                None,
15388            ),
15389            parallel_history_event(
15390                "ActivityScheduled",
15391                3,
15392                "activity_type",
15393                "third",
15394                paths[2].clone(),
15395                None,
15396            ),
15397        ];
15398        for _restart in 0..2 {
15399            let ctx = workflow_context(history.clone());
15400            let mut call = Box::pin(ctx.parallel(nested_parallel_operations()));
15401            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15402            let outcome = call.as_mut().poll(&mut task_context);
15403            assert!(matches!(outcome, Poll::Pending), "{outcome:?}");
15404            assert!(ctx.take_commands().expect("commands").is_empty());
15405        }
15406    }
15407
15408    fn selection_path(index: usize, key: &str) -> Vec<ParallelGroupMetadata> {
15409        vec![selection_group_entry(
15410            1,
15411            2,
15412            index,
15413            "activity",
15414            &SelectionMemberMetadata {
15415                key: SelectionKey::Name(key.to_string()),
15416                index,
15417                base_sequence: index as u64 + 1,
15418                size: 1,
15419                kind: "activity".to_string(),
15420            },
15421        )]
15422    }
15423
15424    fn selection_activity_event(
15425        event_type: &str,
15426        index: usize,
15427        key: &str,
15428        result: Option<Value>,
15429    ) -> HistoryEvent {
15430        let sequence = index as u64 + 1;
15431        let mut event = parallel_history_event(
15432            event_type,
15433            sequence,
15434            "activity_type",
15435            &format!("{key}-activity"),
15436            selection_path(index, key),
15437            result,
15438        );
15439        event.payload["activity_execution_id"] = json!(format!("activity-{key}"));
15440        event.raw.insert(
15441            "id".to_string(),
15442            json!(if event_type == "ActivityCompleted" {
15443                format!("event-{key}")
15444            } else {
15445                format!("{event_type}-{key}")
15446            }),
15447        );
15448        event
15449    }
15450
15451    fn selection_winner_marker() -> HistoryEvent {
15452        history_event(
15453            "SelectionResolved",
15454            json!({
15455                "selection_group_id": "select-calls:1:2",
15456                "selection_group_base_sequence": 1,
15457                "selection_group_size": 2,
15458                "member_key": "fast",
15459                "member_index": 1,
15460                "member_base_sequence": 2,
15461                "member_size": 1,
15462                "operation_kind": "activity",
15463                "operation_identity": "activity-fast",
15464                "outcome": "completed",
15465                "resolution_event_id": "event-fast",
15466                "resolution_event_type": "ActivityCompleted",
15467            }),
15468        )
15469    }
15470
15471    fn keyed_activity_selection(ctx: &WorkflowContext) -> SelectCall {
15472        ctx.select_keyed(vec![
15473            (
15474                "slow",
15475                ParallelOperation::activity_with_options(
15476                    "slow-activity",
15477                    ActivityOptions::new().task_queue("default"),
15478                    json!([]),
15479                ),
15480            ),
15481            (
15482                "fast",
15483                ParallelOperation::activity_with_options(
15484                    "fast-activity",
15485                    ActivityOptions::new().task_queue("default"),
15486                    json!([]),
15487                ),
15488            ),
15489        ])
15490    }
15491
15492    fn assert_persisted_selection_replay(history: Vec<HistoryEvent>) {
15493        let ctx = workflow_context(history);
15494        let mut call = Box::pin(keyed_activity_selection(&ctx));
15495        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15496        let selected = match call.as_mut().poll(&mut task_context) {
15497            Poll::Ready(Ok(selected)) => selected,
15498            Poll::Ready(Err(error)) => panic!("persisted selection winner must replay: {error:?}"),
15499            Poll::Pending => panic!("persisted selection winner must replay without pending"),
15500        };
15501        assert_eq!(selected.key, SelectionKey::Name("fast".to_string()));
15502        assert_eq!(
15503            selected.value,
15504            Some(ParallelResult::Activity(json!("winner-value")))
15505        );
15506        let slow = selected
15507            .handle(&SelectionKey::Name("slow".to_string()))
15508            .expect("slow handle")
15509            .clone();
15510        let mut await_slow = Box::pin(slow.await_result());
15511        assert!(matches!(
15512            await_slow.as_mut().poll(&mut task_context),
15513            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("loser-value")
15514        ));
15515        assert!(ctx.take_commands().expect("commands").is_empty());
15516    }
15517
15518    const SELECTION_COLD_REPLAY_HISTORY: &str = "DURABLE_WORKFLOW_SELECTION_COLD_REPLAY_HISTORY";
15519
15520    fn canonical_selection_history() -> Vec<HistoryEvent> {
15521        const FIXTURE: &[u8] =
15522            include_bytes!("../tests/fixtures/durable_selection_runtime_history.json");
15523        assert_eq!(
15524            format!("{:x}", Sha256::digest(FIXTURE)),
15525            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15526        );
15527        let fixture: Value = serde_json::from_slice(FIXTURE).expect("canonical selection fixture");
15528
15529        serde_json::from_value(fixture["history"].clone()).expect("canonical selection history")
15530    }
15531
15532    #[test]
15533    fn selection_fresh_process_entrypoint() {
15534        let Ok(path) = std::env::var(SELECTION_COLD_REPLAY_HISTORY) else {
15535            return;
15536        };
15537        let persisted = fs::read(path).expect("persisted selection history");
15538        assert_eq!(
15539            format!("{:x}", Sha256::digest(&persisted)),
15540            "51fd8b9c16e978dcef536a5c727b9fdc0ae724d9afc17d9a7837d219f41ee3ba",
15541        );
15542        let fixture: Value =
15543            serde_json::from_slice(&persisted).expect("valid persisted selection fixture");
15544        let history: Vec<HistoryEvent> = serde_json::from_value(fixture["history"].clone())
15545            .expect("valid persisted selection history");
15546
15547        assert_persisted_selection_replay(history);
15548    }
15549
15550    #[test]
15551    fn selection_starts_every_member_with_stable_keys_and_group_identity() {
15552        let ctx = workflow_context(Vec::new());
15553        let mut call = Box::pin(keyed_activity_selection(&ctx));
15554        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15555
15556        assert!(matches!(
15557            call.as_mut().poll(&mut task_context),
15558            Poll::Pending
15559        ));
15560        let commands = ctx.take_commands().expect("selection commands");
15561        assert_eq!(commands.len(), 2);
15562        assert_eq!(commands[0]["selection_member_key"], json!("slow"));
15563        assert_eq!(commands[1]["selection_member_key"], json!("fast"));
15564        assert!(commands.iter().all(|command| {
15565            command["parallel_group_id"] == json!("select-calls:1:2")
15566                && command["parallel_group_mode"] == json!("select")
15567        }));
15568    }
15569
15570    #[test]
15571    fn selection_key_domain_rejects_empty_authoring_and_malformed_history() {
15572        let ctx = workflow_context(Vec::new());
15573        let mut invalid = Box::pin(ctx.select_keyed(vec![(
15574            "",
15575            ParallelOperation::activity("invalid", json!([])),
15576        )]));
15577        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15578        assert!(matches!(
15579            invalid.as_mut().poll(&mut task_context),
15580            Poll::Ready(Err(Error::InvalidParallelGroup(ParallelGroupError {
15581                reason: "selection_key_invalid",
15582                ..
15583            })))
15584        ));
15585
15586        for invalid_key in [json!(""), json!(-1)] {
15587            let mut event = selection_activity_event("ActivityScheduled", 0, "slow", None);
15588            event.payload["selection_member_key"] = invalid_key.clone();
15589            event.payload["parallel_group_path"][0]["selection_member_key"] = invalid_key;
15590            assert!(matches!(
15591                WorkflowState::new_with_identity(
15592                    vec![event],
15593                    None,
15594                    None,
15595                    "rust-workers".to_string(),
15596                    DEFAULT_CODEC.to_string(),
15597                    None,
15598                ),
15599                Err(Error::NonDeterministicReplay(_))
15600            ));
15601        }
15602    }
15603
15604    #[test]
15605    fn selection_preserves_valid_named_and_numeric_keys() {
15606        let ctx = workflow_context(Vec::new());
15607        let mut selection = Box::pin(ctx.select_keyed(vec![
15608            (
15609                SelectionKey::Index(0),
15610                ParallelOperation::activity("numeric", json!([])),
15611            ),
15612            (
15613                SelectionKey::Name("named".to_string()),
15614                ParallelOperation::timer(Duration::from_secs(1)),
15615            ),
15616        ]));
15617        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15618
15619        assert!(matches!(
15620            selection.as_mut().poll(&mut task_context),
15621            Poll::Pending
15622        ));
15623        let commands = ctx.take_commands().expect("selection commands");
15624        assert_eq!(commands[0]["selection_member_key"], json!(0));
15625        assert_eq!(commands[1]["selection_member_key"], json!("named"));
15626    }
15627
15628    #[test]
15629    fn selection_replays_persisted_winner_and_loser_can_be_awaited_later() {
15630        let history = canonical_selection_history();
15631        assert_persisted_selection_replay(history.clone());
15632
15633        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
15634            .join("tests/fixtures/durable_selection_runtime_history.json");
15635        let output =
15636            ProcessCommand::new(std::env::current_exe().expect("current Rust test binary"))
15637                .args([
15638                    "--exact",
15639                    "tests::selection_fresh_process_entrypoint",
15640                    "--nocapture",
15641                ])
15642                .env(SELECTION_COLD_REPLAY_HISTORY, &path)
15643                .output()
15644                .expect("run fresh selection replay process");
15645
15646        assert!(
15647            output.status.success(),
15648            "fresh selection replay failed:\nstdout:\n{}\nstderr:\n{}",
15649            String::from_utf8_lossy(&output.stdout),
15650            String::from_utf8_lossy(&output.stderr),
15651        );
15652    }
15653
15654    #[test]
15655    fn selection_waits_durably_when_terminal_members_precede_the_winner_marker() {
15656        let mut history = canonical_selection_history();
15657        history.retain(|event| event.event_type != "SelectionResolved");
15658        let ctx = workflow_context(history);
15659        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15660        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15661
15662        assert!(matches!(
15663            selection.as_mut().poll(&mut task_context),
15664            Poll::Pending
15665        ));
15666        assert!(ctx.take_commands().expect("commands").is_empty());
15667        assert!(
15668            ctx.matched_recorded_pending()
15669                .expect("selection pending state"),
15670            "terminal member history must keep the workflow durably pending until SelectionResolved commits"
15671        );
15672    }
15673
15674    #[test]
15675    fn selection_terminal_condition_history_waits_durably_for_its_winner_marker() {
15676        for (terminal_event, predicate_satisfied, timeout_seconds) in [
15677            ("ConditionWaitSatisfied", true, None),
15678            ("ConditionWaitTimedOut", false, Some(0)),
15679        ] {
15680            let member = SelectionMemberMetadata {
15681                key: SelectionKey::Name("condition".to_string()),
15682                index: 0,
15683                base_sequence: 1,
15684                size: 1,
15685                kind: "condition".to_string(),
15686            };
15687            let path = vec![selection_group_entry(1, 1, 0, "condition", &member)];
15688            let mut payload = json!({
15689                "sequence": 1,
15690                "condition_wait_id": "condition-1",
15691                "condition_wait_occurrence_id": "rust:condition-wait:0",
15692                "condition_key": "ready",
15693                "condition_definition_fingerprint": "sha256:ready-v1",
15694                "parallel_group_path": path,
15695            });
15696            payload
15697                .as_object_mut()
15698                .expect("condition history payload")
15699                .extend(
15700                    serde_json::to_value(&path[0])
15701                        .expect("condition selection metadata")
15702                        .as_object()
15703                        .expect("condition selection metadata object")
15704                        .clone(),
15705                );
15706            if let Some(timeout_seconds) = timeout_seconds {
15707                payload["timeout_seconds"] = json!(timeout_seconds);
15708            }
15709            let history = vec![
15710                history_event("ConditionWaitOpened", payload.clone()),
15711                history_event(terminal_event, payload),
15712            ];
15713            let ctx = workflow_context(history);
15714            let mut options = ConditionWaitOptions::new("ready", "sha256:ready-v1");
15715            if timeout_seconds.is_some() {
15716                options = options.timeout(Duration::ZERO);
15717            }
15718            let mut selection = Box::pin(ctx.select_keyed(vec![(
15719                "condition",
15720                ParallelOperation::condition(options, move || Ok(predicate_satisfied)),
15721            )]));
15722            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15723
15724            assert!(matches!(
15725                selection.as_mut().poll(&mut task_context),
15726                Poll::Pending
15727            ));
15728            assert!(ctx.take_commands().expect("commands").is_empty());
15729            assert!(
15730                ctx.matched_recorded_pending()
15731                    .expect("condition selection pending state"),
15732                "{terminal_event} must keep the workflow durably pending until SelectionResolved commits"
15733            );
15734        }
15735    }
15736
15737    #[test]
15738    fn selection_immediate_condition_members_open_a_durable_wait() {
15739        for predicate_satisfied in [true, false] {
15740            let ctx = workflow_context(Vec::new());
15741            let mut selection = Box::pin(ctx.select_keyed(vec![(
15742                "condition",
15743                ParallelOperation::condition(
15744                    ConditionWaitOptions::new("ready", "sha256:ready-v1").timeout(Duration::ZERO),
15745                    move || Ok(predicate_satisfied),
15746                ),
15747            )]));
15748            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15749
15750            assert!(matches!(
15751                selection.as_mut().poll(&mut task_context),
15752                Poll::Pending
15753            ));
15754            let commands = ctx.take_commands().expect("condition selection command");
15755            assert_eq!(commands.len(), 1);
15756            assert_eq!(commands[0]["type"], json!("open_condition_wait"));
15757            assert_eq!(commands[0]["timeout_seconds"], json!(0));
15758            assert_eq!(
15759                commands[0]["parallel_group_path"][0]["parallel_group_mode"],
15760                json!("select")
15761            );
15762        }
15763    }
15764
15765    #[test]
15766    fn selection_loser_cancellation_is_explicit_and_idempotent() {
15767        let history = vec![
15768            selection_activity_event("ActivityScheduled", 0, "slow", None),
15769            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15770            selection_winner_marker(),
15771        ];
15772        let ctx = workflow_context(history.clone());
15773        let mut call = Box::pin(keyed_activity_selection(&ctx));
15774        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15775        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15776            panic!("winner must replay");
15777        };
15778        let slow = selected
15779            .handle(&SelectionKey::Name("slow".to_string()))
15780            .expect("slow handle")
15781            .clone();
15782        let mut cancel = Box::pin(slow.cancel());
15783        assert!(matches!(
15784            cancel.as_mut().poll(&mut task_context),
15785            Poll::Pending
15786        ));
15787        assert!(matches!(
15788            cancel.as_mut().poll(&mut task_context),
15789            Poll::Pending
15790        ));
15791        let commands = ctx.take_commands().expect("cancel command");
15792        assert_eq!(commands.len(), 1);
15793        assert_eq!(commands[0]["type"], json!("cancel_selection_operation"));
15794        assert_eq!(commands[0]["member_key"], json!("slow"));
15795
15796        let mut cancelled_history = history;
15797        cancelled_history.push(history_event(
15798            "SelectionOperationCancelled",
15799            json!({
15800                "selection_group_id": "select-calls:1:2",
15801                "member_key": "slow",
15802                "member_index": 0,
15803                "member_base_sequence": 1,
15804                "member_size": 1,
15805                "operation_kind": "activity",
15806                "operation_identity": "activity-slow",
15807                "cancelled_at": "2026-08-27T00:00:00Z",
15808            }),
15809        ));
15810        let replayed = workflow_context(cancelled_history);
15811        let mut call = Box::pin(keyed_activity_selection(&replayed));
15812        let Poll::Ready(Ok(selected)) = call.as_mut().poll(&mut task_context) else {
15813            panic!("winner must replay after cancellation");
15814        };
15815        let slow = selected
15816            .handle(&SelectionKey::Name("slow".to_string()))
15817            .expect("slow handle")
15818            .clone();
15819        let mut cancel = Box::pin(slow.cancel());
15820        assert!(matches!(
15821            cancel.as_mut().poll(&mut task_context),
15822            Poll::Ready(Ok(()))
15823        ));
15824        assert!(replayed.take_commands().expect("commands").is_empty());
15825    }
15826
15827    #[test]
15828    fn selection_cancellation_marker_is_bound_to_every_authored_handle_field() {
15829        let base_history = vec![
15830            selection_activity_event("ActivityScheduled", 0, "slow", None),
15831            selection_activity_event("ActivityScheduled", 1, "fast", None),
15832            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15833            selection_winner_marker(),
15834        ];
15835        for (field, corrupt) in [
15836            ("member_key", json!("fast")),
15837            ("member_index", json!(1)),
15838            ("member_base_sequence", json!(3)),
15839            ("member_size", json!(2)),
15840            ("operation_kind", json!("timer")),
15841            ("operation_identity", json!("forged")),
15842        ] {
15843            let mut cancellation = json!({
15844                "selection_group_id": "select-calls:1:2",
15845                "member_key": "slow",
15846                "member_index": 0,
15847                "member_base_sequence": 1,
15848                "member_size": 1,
15849                "operation_kind": "activity",
15850                "operation_identity": "activity-slow",
15851            });
15852            cancellation[field] = corrupt;
15853            let mut history = base_history.clone();
15854            history.push(history_event("SelectionOperationCancelled", cancellation));
15855            let ctx = workflow_context(history);
15856            let mut selection = Box::pin(keyed_activity_selection(&ctx));
15857            let mut task_context = TaskContext::from_waker(noop_waker_ref());
15858
15859            assert!(matches!(
15860                selection.as_mut().poll(&mut task_context),
15861                Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15862            ));
15863        }
15864    }
15865
15866    #[test]
15867    fn selection_child_identity_prefers_the_durable_run_id() {
15868        let ctx = workflow_context(vec![history_event(
15869            "ChildWorkflowScheduled",
15870            json!({
15871                "sequence": 1,
15872                "child_workflow_type": "child",
15873                "child_workflow_instance_id": "child-instance",
15874                "child_workflow_run_id": "child-run",
15875            }),
15876        )]);
15877        let state = ctx.state.lock().expect("workflow state");
15878
15879        assert_eq!(
15880            selection_operation_identity(&state, "child", 1, 1),
15881            "child-run"
15882        );
15883    }
15884
15885    #[test]
15886    fn selection_activity_identity_requires_canonical_execution_id() {
15887        let slow = selection_activity_event("ActivityScheduled", 0, "slow", None);
15888        let mut fast_open = selection_activity_event("ActivityScheduled", 1, "fast", None);
15889        let mut fast_completed =
15890            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner")));
15891        for event in [&mut fast_open, &mut fast_completed] {
15892            event
15893                .payload
15894                .as_object_mut()
15895                .expect("activity payload")
15896                .remove("activity_execution_id");
15897            event.payload["activity_id"] = json!("forged-activity-id");
15898        }
15899        let mut marker = selection_winner_marker();
15900        marker.payload["operation_identity"] = json!("forged-activity-id");
15901        let ctx = workflow_context(vec![slow, fast_open, fast_completed, marker]);
15902        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15903        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15904
15905        assert!(matches!(
15906            selection.as_mut().poll(&mut task_context),
15907            Poll::Ready(Err(Error::NonDeterministicReplay(_)))
15908        ));
15909    }
15910
15911    #[test]
15912    fn selection_completion_before_cancellation_remains_awaitable() {
15913        let history = vec![
15914            selection_activity_event("ActivityScheduled", 0, "slow", None),
15915            selection_activity_event("ActivityCompleted", 1, "fast", Some(json!("winner"))),
15916            selection_winner_marker(),
15917            selection_activity_event(
15918                "ActivityCompleted",
15919                0,
15920                "slow",
15921                Some(json!("completed-first")),
15922            ),
15923        ];
15924        let ctx = workflow_context(history);
15925        let mut selection = Box::pin(keyed_activity_selection(&ctx));
15926        let mut task_context = TaskContext::from_waker(noop_waker_ref());
15927        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
15928            panic!("winner must replay");
15929        };
15930        let slow = selected
15931            .handle(&SelectionKey::Name("slow".to_string()))
15932            .expect("slow handle")
15933            .clone();
15934        let mut cancel = Box::pin(slow.cancel());
15935        assert!(matches!(
15936            cancel.as_mut().poll(&mut task_context),
15937            Poll::Ready(Ok(()))
15938        ));
15939        let mut await_slow = Box::pin(slow.await_result());
15940        assert!(matches!(
15941            await_slow.as_mut().poll(&mut task_context),
15942            Poll::Ready(Ok(ParallelResult::Activity(value))) if value == json!("completed-first")
15943        ));
15944        let commands = ctx.take_commands().expect("commands");
15945        assert!(commands.is_empty());
15946    }
15947
15948    #[test]
15949    fn selection_nested_later_failure_before_cancel_remains_the_awaited_failure() {
15950        let nested_member = SelectionMemberMetadata {
15951            key: SelectionKey::Name("nested".to_string()),
15952            index: 0,
15953            base_sequence: 1,
15954            size: 2,
15955            kind: "group".to_string(),
15956        };
15957        let deadline_member = SelectionMemberMetadata {
15958            key: SelectionKey::Name("deadline".to_string()),
15959            index: 1,
15960            base_sequence: 3,
15961            size: 1,
15962            kind: "timer".to_string(),
15963        };
15964        let nested_paths = [
15965            vec![
15966                selection_group_entry(1, 3, 0, "mixed", &nested_member),
15967                parallel_group_entry(1, 2, 0, "activity"),
15968            ],
15969            vec![
15970                selection_group_entry(1, 3, 1, "mixed", &nested_member),
15971                parallel_group_entry(1, 2, 1, "activity"),
15972            ],
15973        ];
15974        let deadline_path = vec![selection_group_entry(1, 3, 2, "mixed", &deadline_member)];
15975        let mut timer_fired = parallel_history_event(
15976            "TimerFired",
15977            3,
15978            "timer_id",
15979            "timer-3",
15980            deadline_path.clone(),
15981            None,
15982        );
15983        timer_fired.payload["delay_seconds"] = json!(0);
15984        timer_fired
15985            .raw
15986            .insert("id".to_string(), json!("timer-fired"));
15987        let mut timer_scheduled = parallel_history_event(
15988            "TimerScheduled",
15989            3,
15990            "timer_id",
15991            "timer-3",
15992            deadline_path,
15993            None,
15994        );
15995        timer_scheduled.payload["delay_seconds"] = json!(0);
15996        let history = vec![
15997            parallel_history_event(
15998                "ActivityScheduled",
15999                1,
16000                "activity_type",
16001                "nested-first",
16002                nested_paths[0].clone(),
16003                None,
16004            ),
16005            parallel_history_event(
16006                "ActivityScheduled",
16007                2,
16008                "activity_type",
16009                "nested-second",
16010                nested_paths[1].clone(),
16011                None,
16012            ),
16013            timer_scheduled,
16014            timer_fired,
16015            history_event(
16016                "SelectionResolved",
16017                json!({
16018                    "selection_group_id": "select-calls:1:3",
16019                    "selection_group_base_sequence": 1,
16020                    "selection_group_size": 3,
16021                    "member_key": "deadline",
16022                    "member_index": 1,
16023                    "member_base_sequence": 3,
16024                    "member_size": 1,
16025                    "operation_kind": "timer",
16026                    "operation_identity": "timer-3",
16027                    "outcome": "completed",
16028                    "resolution_event_id": "timer-fired",
16029                    "resolution_event_type": "TimerFired",
16030                }),
16031            ),
16032            parallel_history_event(
16033                "ActivityFailed",
16034                2,
16035                "activity_type",
16036                "nested-second",
16037                nested_paths[1].clone(),
16038                None,
16039            ),
16040        ];
16041        let ctx = workflow_context(history);
16042        let mut selection = Box::pin(ctx.select_keyed(vec![
16043            (
16044                "nested",
16045                ParallelOperation::group(vec![
16046                    ParallelOperation::activity("nested-first", json!([])),
16047                    ParallelOperation::activity("nested-second", json!([])),
16048                ]),
16049            ),
16050            ("deadline", ParallelOperation::timer(Duration::ZERO)),
16051        ]));
16052        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16053        let Poll::Ready(Ok(selected)) = selection.as_mut().poll(&mut task_context) else {
16054            panic!("deadline winner must replay");
16055        };
16056        let nested = selected
16057            .handle(&SelectionKey::Name("nested".to_string()))
16058            .expect("nested handle")
16059            .clone();
16060        let mut cancel = Box::pin(nested.cancel());
16061        assert!(matches!(
16062            cancel.as_mut().poll(&mut task_context),
16063            Poll::Ready(Ok(()))
16064        ));
16065        let mut await_nested = Box::pin(nested.await_result());
16066
16067        assert!(matches!(
16068            await_nested.as_mut().poll(&mut task_context),
16069            Poll::Ready(Err(Error::ActivityFailed(_)))
16070        ));
16071        assert!(ctx.take_commands().expect("commands").is_empty());
16072    }
16073
16074    #[test]
16075    fn selection_supports_child_timer_signal_condition_and_nested_groups() {
16076        let ctx = workflow_context(Vec::new());
16077        let mut call = Box::pin(ctx.select(vec![
16078            ParallelOperation::child_workflow(
16079                "child",
16080                ChildWorkflowOptions::new("children"),
16081                json!([]),
16082            ),
16083            ParallelOperation::timer(Duration::from_secs(30)),
16084            ParallelOperation::signal("approval"),
16085            ParallelOperation::condition(
16086                ConditionWaitOptions::new("ready", "sha256:ready"),
16087                || Ok(false),
16088            ),
16089            ParallelOperation::group(vec![
16090                ParallelOperation::activity("nested-one", json!([])),
16091                ParallelOperation::activity("nested-two", json!([])),
16092            ]),
16093        ]));
16094        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16095        assert!(matches!(
16096            call.as_mut().poll(&mut task_context),
16097            Poll::Pending
16098        ));
16099        let commands = ctx.take_commands().expect("selection commands");
16100        assert_eq!(
16101            commands
16102                .iter()
16103                .map(|command| command["type"].as_str().unwrap_or_default())
16104                .collect::<Vec<_>>(),
16105            [
16106                "start_child_workflow",
16107                "start_timer",
16108                "open_signal_wait",
16109                "open_condition_wait",
16110                "schedule_activity",
16111                "schedule_activity",
16112            ]
16113        );
16114        assert!(commands.iter().all(|command| {
16115            command["parallel_group_path"][0]["parallel_group_mode"] == json!("select")
16116        }));
16117        assert_eq!(
16118            commands[4]["parallel_group_path"].as_array().map(Vec::len),
16119            Some(2)
16120        );
16121        assert_eq!(
16122            commands[4]["parallel_group_path"][0]["selection_member_kind"],
16123            json!("group")
16124        );
16125        assert_eq!(
16126            commands[5]["parallel_group_path"][0]["selection_member_kind"],
16127            json!("group")
16128        );
16129
16130        let one_leaf_ctx = workflow_context(Vec::new());
16131        let mut one_leaf = Box::pin(one_leaf_ctx.select(vec![ParallelOperation::group(vec![
16132            ParallelOperation::activity("nested-only", json!([])),
16133        ])]));
16134        assert!(matches!(
16135            one_leaf.as_mut().poll(&mut task_context),
16136            Poll::Pending
16137        ));
16138        let one_leaf_commands = one_leaf_ctx.take_commands().expect("one-leaf commands");
16139        assert_eq!(one_leaf_commands.len(), 1);
16140        assert_eq!(
16141            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_kind"],
16142            json!("group")
16143        );
16144        assert_eq!(
16145            one_leaf_commands[0]["parallel_group_path"][0]["selection_member_size"],
16146            json!(1)
16147        );
16148    }
16149
16150    async fn trip_saga(ctx: WorkflowContext) -> Result<Value> {
16151        let mut saga = ctx.saga();
16152        let outcome = async {
16153            let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16154            saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16155            let hotel = ctx.activity("trip.reserve-hotel", json!([])).await?;
16156            saga.add_compensation("trip.cancel-hotel", json!([hotel]))?;
16157            ctx.activity("trip.charge", json!([])).await?;
16158            Ok(json!({"status": "booked"}))
16159        }
16160        .await;
16161        saga.finish(outcome).await
16162    }
16163
16164    fn saga_activity(
16165        event_type: &str,
16166        sequence: u64,
16167        activity_type: &str,
16168        result: Option<Value>,
16169    ) -> HistoryEvent {
16170        let mut payload = json!({
16171            "sequence": sequence,
16172            "activity_type": activity_type,
16173            "message": format!("{activity_type} failed"),
16174            "exception_type": "PlannedFailure",
16175            "non_retryable": true,
16176        });
16177        if let Some(result) = result {
16178            payload["result"] = fixture_envelope(result);
16179        }
16180        history_event(event_type, payload)
16181    }
16182
16183    #[test]
16184    fn saga_replays_reverse_compensation_across_restart_and_duplicate_delivery() {
16185        let completed_hotel_compensation = saga_activity(
16186            "ActivityCompleted",
16187            4,
16188            "trip.cancel-hotel",
16189            Some(Value::Null),
16190        );
16191        let history = vec![
16192            saga_activity(
16193                "ActivityCompleted",
16194                1,
16195                "trip.reserve-flight",
16196                Some(json!("flight-1")),
16197            ),
16198            saga_activity(
16199                "ActivityCompleted",
16200                2,
16201                "trip.reserve-hotel",
16202                Some(json!("hotel-1")),
16203            ),
16204            saga_activity("ActivityFailed", 3, "trip.charge", None),
16205            completed_hotel_compensation.clone(),
16206            completed_hotel_compensation,
16207        ];
16208
16209        for _restart in 0..2 {
16210            let ctx = workflow_context(history.clone());
16211            let mut future = Box::pin(trip_saga(ctx.clone()));
16212            let mut task_context = TaskContext::from_waker(noop_waker_ref());
16213            assert!(matches!(
16214                future.as_mut().poll(&mut task_context),
16215                Poll::Pending
16216            ));
16217            let commands = ctx.take_commands().expect("compensation command");
16218            assert_eq!(commands.len(), 1);
16219            assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16220        }
16221    }
16222
16223    #[test]
16224    fn saga_compensation_failure_preserves_both_typed_failures() {
16225        let history = vec![
16226            saga_activity(
16227                "ActivityCompleted",
16228                1,
16229                "trip.reserve-flight",
16230                Some(json!("flight-1")),
16231            ),
16232            saga_activity(
16233                "ActivityCompleted",
16234                2,
16235                "trip.reserve-hotel",
16236                Some(json!("hotel-1")),
16237            ),
16238            saga_activity("ActivityFailed", 3, "trip.charge", None),
16239            saga_activity("ActivityFailed", 4, "trip.cancel-hotel", None),
16240        ];
16241        let ctx = workflow_context(history);
16242        let mut future = Box::pin(trip_saga(ctx));
16243        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16244        let Poll::Ready(Err(Error::SagaCompensationFailed(failure))) =
16245            future.as_mut().poll(&mut task_context)
16246        else {
16247            panic!("compensation failure must remain structured");
16248        };
16249        assert!(matches!(
16250            *failure.initiating_failure,
16251            Error::ActivityFailed(_)
16252        ));
16253        assert!(matches!(
16254            *failure.compensation_failure,
16255            Error::ActivityFailed(_)
16256        ));
16257        assert_eq!(failure.compensation_activity_type, "trip.cancel-hotel");
16258        assert_eq!(failure.compensation_registration_order, 2);
16259    }
16260
16261    #[test]
16262    fn saga_compensates_cooperative_cancellation() {
16263        let ctx = workflow_context(vec![saga_activity(
16264            "ActivityCompleted",
16265            1,
16266            "trip.reserve-flight",
16267            Some(json!("flight-1")),
16268        )]);
16269        ctx.state.lock().expect("state").cancel_requested = true;
16270        let run = {
16271            let ctx = ctx.clone();
16272            async move {
16273                let mut saga = ctx.saga();
16274                let outcome = async {
16275                    let flight = ctx.activity("trip.reserve-flight", json!([])).await?;
16276                    saga.add_compensation("trip.cancel-flight", json!([flight]))?;
16277                    ctx.throw_if_cancellation_requested()?;
16278                    Ok(json!("unexpected"))
16279                }
16280                .await;
16281                saga.finish(outcome).await
16282            }
16283        };
16284        let mut future = Box::pin(run);
16285        let mut task_context = TaskContext::from_waker(noop_waker_ref());
16286        assert!(matches!(
16287            future.as_mut().poll(&mut task_context),
16288            Poll::Pending
16289        ));
16290        let commands = ctx.take_commands().expect("cancellation compensation");
16291        assert_eq!(commands[0]["activity_type"], "trip.cancel-flight");
16292    }
16293
16294    fn workflow_task(
16295        workflow_type: &str,
16296        history_events: Vec<HistoryEvent>,
16297        payload_codec: &str,
16298    ) -> WorkflowTask {
16299        WorkflowTask {
16300            task_id: format!("wft-{workflow_type}"),
16301            workflow_command_id: None,
16302            workflow_id: Some(format!("wf-{workflow_type}")),
16303            run_id: Some(format!("run-{workflow_type}")),
16304            workflow_type: workflow_type.to_string(),
16305            cancel_requested: false,
16306            payload_codec: payload_codec.to_string(),
16307            arguments: Some(
16308                encode_value_envelope(&json!([]), payload_codec).expect("workflow arguments"),
16309            ),
16310            total_history_events: Some(history_events.len() as u64),
16311            history_size_bytes: None,
16312            continue_as_new_recommended: None,
16313            history_budget_pressure: None,
16314            history_events,
16315            next_history_page_token: None,
16316            workflow_task_attempt: 1,
16317            workflow_signal_id: None,
16318            signal_name: None,
16319            signal_arguments: None,
16320            workflow_update_id: None,
16321            update_name: None,
16322            lease_owner: Some("rust-worker".to_string()),
16323        }
16324    }
16325
16326    #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
16327    struct SideEffectProbe {
16328        request_id: String,
16329        attempt: u32,
16330    }
16331
16332    #[test]
16333    fn typed_side_effect_runs_callback_once_and_replay_skips_it() {
16334        let calls = AtomicUsize::new(0);
16335        let ctx = workflow_context(Vec::new());
16336        let value = ctx
16337            .side_effect(|| {
16338                calls.fetch_add(1, Ordering::SeqCst);
16339                SideEffectProbe {
16340                    request_id: "request-42".to_string(),
16341                    attempt: 3,
16342                }
16343            })
16344            .expect("first side effect");
16345        assert_eq!(value.attempt, 3);
16346        assert_eq!(calls.load(Ordering::SeqCst), 1);
16347        let commands = ctx.take_commands().expect("commands");
16348        assert_eq!(commands.len(), 1);
16349        assert_eq!(commands[0]["type"], "record_side_effect");
16350        assert_eq!(
16351            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16352            serde_json::to_value(&value).expect("value")
16353        );
16354
16355        let replay = workflow_context(vec![history_event(
16356            "SideEffectRecorded",
16357            json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16358        )]);
16359        let replayed: SideEffectProbe = replay
16360            .side_effect(|| {
16361                calls.fetch_add(1, Ordering::SeqCst);
16362                panic!("committed side-effect callbacks must not run during replay")
16363            })
16364            .expect("replayed side effect");
16365        assert_eq!(replayed, value);
16366        assert_eq!(calls.load(Ordering::SeqCst), 1);
16367        assert!(replay.take_commands().expect("commands").is_empty());
16368        replay.ensure_history_consumed().expect("history consumed");
16369    }
16370
16371    #[test]
16372    fn side_effect_uses_avro_envelope_and_uuid_is_replay_stable() {
16373        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16374        let value = ctx
16375            .side_effect(|| SideEffectProbe {
16376                request_id: "avro-request".to_string(),
16377                attempt: 1,
16378            })
16379            .expect("Avro side effect");
16380        let uuid = ctx.uuid_v4().expect("deterministic UUID");
16381        let commands = ctx.take_commands().expect("commands");
16382        assert_eq!(commands.len(), 2);
16383        assert_eq!(commands[0]["result"]["codec"], DEFAULT_CODEC);
16384        assert_eq!(commands[1]["result"]["codec"], DEFAULT_CODEC);
16385        assert_eq!(
16386            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("Avro result"),
16387            serde_json::to_value(&value).expect("value")
16388        );
16389
16390        let replay = workflow_context_with_codec(
16391            vec![
16392                history_event(
16393                    "SideEffectRecorded",
16394                    json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16395                ),
16396                history_event(
16397                    "SideEffectRecorded",
16398                    json!({"sequence": 2, "result": commands[1]["result"].clone()}),
16399                ),
16400            ],
16401            DEFAULT_CODEC,
16402        );
16403        let replayed: SideEffectProbe = replay
16404            .side_effect(|| panic!("Avro callback must not run"))
16405            .expect("replayed Avro value");
16406        let replayed_uuid = replay.uuid_v4().expect("replayed UUID");
16407        assert_eq!(replayed, value);
16408        assert_eq!(replayed_uuid, uuid);
16409        assert!(replay.take_commands().expect("commands").is_empty());
16410    }
16411
16412    #[test]
16413    fn typed_side_effect_replay_preserves_bytes_and_maps() {
16414        let ctx = workflow_context_with_codec(Vec::new(), DEFAULT_CODEC);
16415        let value = ctx
16416            .side_effect_avro_value(typed_fidelity_probe)
16417            .expect("typed side effect");
16418        let commands = ctx.take_commands().expect("side-effect command");
16419        assert_eq!(
16420            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
16421                .expect("recorded side effect"),
16422            value
16423        );
16424
16425        let replay = workflow_context_with_codec(
16426            vec![history_event(
16427                "SideEffectRecorded",
16428                json!({"sequence": 1, "result": commands[0]["result"].clone()}),
16429            )],
16430            DEFAULT_CODEC,
16431        );
16432        assert_eq!(
16433            replay
16434                .side_effect_avro_value(|| panic!("replay must not invoke callback"))
16435                .expect("replayed typed side effect"),
16436            value
16437        );
16438    }
16439
16440    #[test]
16441    fn ordered_side_effects_share_the_durable_command_stream() {
16442        let first = encode_value_envelope(&json!("first"), DEFAULT_CODEC).expect("first");
16443        let second = encode_value_envelope(&json!(29), DEFAULT_CODEC).expect("second");
16444        let ctx = workflow_context(vec![
16445            history_event(
16446                "SideEffectRecorded",
16447                json!({"sequence": 1, "result": first}),
16448            ),
16449            history_event(
16450                "SideEffectRecorded",
16451                json!({"sequence": 2, "result": second}),
16452            ),
16453        ]);
16454        let first: String = ctx
16455            .side_effect(|| panic!("first callback must not run"))
16456            .expect("first replay");
16457        let second: i32 = ctx
16458            .side_effect(|| panic!("second callback must not run"))
16459            .expect("second replay");
16460        assert_eq!(first, "first");
16461        assert_eq!(second, 29);
16462        ctx.ensure_history_consumed().expect("ordered history");
16463
16464        let reordered = workflow_context(vec![history_event(
16465            "VersionMarkerRecorded",
16466            json!({
16467                "sequence": 1,
16468                "change_id": "before-side-effect",
16469                "version": 1,
16470                "min_supported": 1,
16471                "max_supported": 1,
16472            }),
16473        )]);
16474        let error = reordered
16475            .side_effect(|| "new".to_string())
16476            .expect_err("command reordering must fail");
16477        assert!(matches!(
16478            error,
16479            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16480                if reason == "recorded_command_mismatch"
16481        ));
16482    }
16483
16484    #[test]
16485    fn version_markers_replay_across_upgrades_and_do_not_duplicate() {
16486        let ctx = workflow_context(Vec::new());
16487        assert_eq!(ctx.get_version("checkout-v2", 1, 2).expect("version"), 2);
16488        assert_eq!(ctx.get_version("checkout-v2", 1, 3).expect("cached"), 2);
16489        assert!(ctx.patched("new-search").expect("patch"));
16490        ctx.deprecate_patch("new-search").expect("deprecate patch");
16491        let commands = ctx.take_commands().expect("commands");
16492        assert_eq!(commands.len(), 2);
16493        assert_eq!(commands[0]["type"], "record_version_marker");
16494        assert_eq!(commands[0]["version"], 2);
16495        assert_eq!(commands[1]["change_id"], "new-search");
16496
16497        let replay = workflow_context(vec![history_event(
16498            "VersionMarkerRecorded",
16499            json!({
16500                "sequence": 1,
16501                "change_id": "checkout-v2",
16502                "version": 2,
16503                "min_supported": 1,
16504                "max_supported": 2,
16505            }),
16506        )]);
16507        assert_eq!(replay.get_version("checkout-v2", 1, 4).expect("upgrade"), 2);
16508        assert_eq!(replay.get_version("checkout-v2", 2, 5).expect("repeat"), 2);
16509        assert!(replay.take_commands().expect("commands").is_empty());
16510        replay.ensure_history_consumed().expect("history consumed");
16511    }
16512
16513    #[test]
16514    fn version_markers_reject_incompatible_or_malformed_history() {
16515        let incompatible = workflow_context(vec![history_event(
16516            "VersionMarkerRecorded",
16517            json!({
16518                "sequence": 1,
16519                "change_id": "checkout-v2",
16520                "version": 1,
16521                "min_supported": 1,
16522                "max_supported": 2,
16523            }),
16524        )]);
16525        let error = incompatible
16526            .get_version("checkout-v2", 2, 3)
16527            .expect_err("old version is unsupported");
16528        assert!(matches!(
16529            error,
16530            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16531                if reason == "version_marker_incompatible_range"
16532        ));
16533
16534        for (history, reason) in [
16535            (
16536                vec![history_event("SideEffectRecorded", json!({"sequence": 1}))],
16537                "side_effect_result_missing",
16538            ),
16539            (
16540                vec![history_event(
16541                    "SideEffectRecorded",
16542                    json!({
16543                        "sequence": 1,
16544                        "result": {"codec": "avro", "blob": "not-base64"},
16545                    }),
16546                )],
16547                "side_effect_payload_incompatible",
16548            ),
16549            (
16550                vec![history_event(
16551                    "SideEffectRecorded",
16552                    json!({"sequence": 1, "result": {"unwrapped": true}}),
16553                )],
16554                "side_effect_payload_malformed",
16555            ),
16556            (
16557                vec![history_event(
16558                    "VersionMarkerRecorded",
16559                    json!({
16560                        "sequence": 1,
16561                        "change_id": "change",
16562                        "version": 1,
16563                        "min_supported": 2,
16564                        "max_supported": 1,
16565                    }),
16566                )],
16567                "version_marker_history_range_invalid",
16568            ),
16569        ] {
16570            let error = WorkflowState::new(
16571                history,
16572                "rust-workers".to_string(),
16573                DEFAULT_CODEC.to_string(),
16574                None,
16575            )
16576            .expect_err("malformed history must fail");
16577            assert!(matches!(
16578                error,
16579                Error::NonDeterministicReplay(ReplayFailure { reason: actual, .. })
16580                    if actual == reason
16581            ));
16582        }
16583    }
16584
16585    #[test]
16586    fn typed_search_attributes_replay_value_and_type_identity_after_restart() {
16587        let history = vec![history_event(
16588            "SearchAttributesUpserted",
16589            json!({
16590                "sequence": 1,
16591                "attributes": {"customer_tier": "gold"},
16592                "attribute_types": {"customer_tier": "keyword"},
16593                "merged": {"customer_tier": "gold"}
16594            }),
16595        )];
16596
16597        let matching = workflow_context(history.clone());
16598        matching
16599            .upsert_search_attributes(
16600                SearchAttributeUpdate::new()
16601                    .keyword("customer_tier", "gold")
16602                    .expect("keyword update"),
16603            )
16604            .expect("matching typed update must replay");
16605        matching
16606            .ensure_history_consumed()
16607            .expect("history consumed");
16608
16609        let changed_type = workflow_context(history.clone());
16610        let error = changed_type
16611            .upsert_search_attributes(
16612                SearchAttributeUpdate::new()
16613                    .string("customer_tier", "gold")
16614                    .expect("string update"),
16615            )
16616            .expect_err("same JSON value with a different declaration must be nondeterministic");
16617        let Error::NonDeterministicReplay(failure) = error else {
16618            panic!("typed identity drift must be a replay failure");
16619        };
16620        assert_eq!(failure.reason, "search_attribute_type_mismatch");
16621        assert_eq!(failure.sequence, Some(1));
16622
16623        let changed_value = workflow_context(history);
16624        let error = changed_value
16625            .upsert_search_attributes(
16626                SearchAttributeUpdate::new()
16627                    .keyword("customer_tier", "platinum")
16628                    .expect("keyword update"),
16629            )
16630            .expect_err("changed values must be nondeterministic");
16631        let Error::NonDeterministicReplay(failure) = error else {
16632            panic!("value drift must be a replay failure");
16633        };
16634        assert_eq!(failure.reason, "search_attribute_value_mismatch");
16635    }
16636
16637    #[test]
16638    fn legacy_search_attribute_history_keeps_type_identity_unknown() {
16639        let history = vec![history_event(
16640            "SearchAttributesUpserted",
16641            json!({
16642                "sequence": 1,
16643                "attributes": {"customer_tier": "gold"},
16644                "merged": {"customer_tier": "gold"}
16645            }),
16646        )];
16647
16648        for update in [
16649            SearchAttributeUpdate::new()
16650                .keyword("customer_tier", "gold")
16651                .expect("keyword update"),
16652            SearchAttributeUpdate::new()
16653                .string("customer_tier", "gold")
16654                .expect("string update"),
16655        ] {
16656            let restarted = workflow_context(history.clone());
16657            restarted
16658                .upsert_search_attributes(update)
16659                .expect("legacy history constrains values but has unknown type identity");
16660            restarted
16661                .ensure_history_consumed()
16662                .expect("history consumed");
16663        }
16664    }
16665
16666    #[test]
16667    fn search_attribute_command_emits_canonical_types() {
16668        let ctx = workflow_context(Vec::new());
16669        ctx.upsert_search_attributes(
16670            SearchAttributeUpdate::new()
16671                .keyword("customer_tier", "gold")
16672                .expect("keyword update")
16673                .int("attempts", 3)
16674                .expect("int update")
16675                .delete("obsolete")
16676                .expect("delete update"),
16677        )
16678        .expect("valid search attributes");
16679
16680        assert_eq!(
16681            ctx.take_commands().expect("commands"),
16682            vec![json!({
16683                "type": "upsert_search_attributes",
16684                "attributes": {
16685                    "attempts": 3,
16686                    "customer_tier": "gold",
16687                    "obsolete": null
16688                },
16689                "attribute_types": {
16690                    "attempts": "int",
16691                    "customer_tier": "keyword"
16692                }
16693            })]
16694        );
16695    }
16696
16697    #[test]
16698    fn duplicate_side_effects_and_version_markers_are_rejected() {
16699        let duplicate_side_effect = WorkflowState::new(
16700            vec![
16701                history_event(
16702                    "SideEffectRecorded",
16703                    json!({"sequence": 1, "result": fixture_envelope(json!(1))}),
16704                ),
16705                history_event(
16706                    "SideEffectRecorded",
16707                    json!({"sequence": 1, "result": fixture_envelope(json!(2))}),
16708                ),
16709            ],
16710            "rust-workers".to_string(),
16711            DEFAULT_CODEC.to_string(),
16712            None,
16713        )
16714        .expect_err("duplicate side effect");
16715        assert!(matches!(
16716            duplicate_side_effect,
16717            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16718                if reason == "duplicate_side_effect_record"
16719        ));
16720
16721        let marker = |sequence| {
16722            history_event(
16723                "VersionMarkerRecorded",
16724                json!({
16725                    "sequence": sequence,
16726                    "change_id": "same-change",
16727                    "version": 1,
16728                    "min_supported": 1,
16729                    "max_supported": 1,
16730                }),
16731            )
16732        };
16733        let duplicate_marker = WorkflowState::new(
16734            vec![marker(1), marker(3)],
16735            "rust-workers".to_string(),
16736            DEFAULT_CODEC.to_string(),
16737            None,
16738        )
16739        .expect_err("duplicate marker");
16740        assert!(matches!(
16741            duplicate_marker,
16742            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16743                if reason == "duplicate_version_marker"
16744        ));
16745    }
16746
16747    #[test]
16748    fn workflow_stream_authoring_derives_identity_and_replay_skips_duplicate_append() {
16749        let mut state = WorkflowState::new(
16750            Vec::new(),
16751            "rust-workers".to_string(),
16752            DEFAULT_CODEC.to_string(),
16753            None,
16754        )
16755        .expect("workflow state");
16756        state.workflow_command_identity = "command-7".to_string();
16757        let context = WorkflowContext {
16758            state: Arc::new(Mutex::new(state)),
16759        };
16760        let item =
16761            WorkflowStreamAppendItem::from_reference("s3://bucket/item.avro").item_type("receipt");
16762
16763        context
16764            .append_workflow_stream("output", &[item], Some(10))
16765            .expect("append command");
16766        context
16767            .error_workflow_stream("output", "producer failed", None)
16768            .expect("error command");
16769        let commands = context.take_commands().expect("commands");
16770
16771        assert_eq!(commands[0]["type"], "record_side_effect");
16772        assert_eq!(
16773            commands[0]["workflow_stream"]["command_identity"],
16774            "command-7"
16775        );
16776        assert_eq!(commands[0]["workflow_stream"]["command_ordinal"], 0);
16777        assert_eq!(
16778            commands[0]["workflow_stream"]["items"][0]["idempotency_key"],
16779            "dw-stream:command-7:0:0"
16780        );
16781        assert_eq!(commands[1]["workflow_stream"]["operation"], "error");
16782
16783        let recorded = history_event(
16784            "SideEffectRecorded",
16785            json!({"sequence": 1, "result": fixture_envelope(Value::Null)}),
16786        );
16787        let mut replay_state = WorkflowState::new(
16788            vec![recorded],
16789            "rust-workers".to_string(),
16790            DEFAULT_CODEC.to_string(),
16791            None,
16792        )
16793        .expect("replay state");
16794        replay_state.workflow_command_identity = "command-7".to_string();
16795        let replay_context = WorkflowContext {
16796            state: Arc::new(Mutex::new(replay_state)),
16797        };
16798        replay_context
16799            .append_workflow_stream(
16800                "output",
16801                &[WorkflowStreamAppendItem::from_reference(
16802                    "s3://bucket/item.avro",
16803                )],
16804                Some(10),
16805            )
16806            .expect("replayed append");
16807        assert!(replay_context
16808            .take_commands()
16809            .expect("replayed commands")
16810            .is_empty());
16811    }
16812
16813    #[test]
16814    fn workflow_stream_authoring_requires_server_durable_command_identity() {
16815        let context = workflow_context(Vec::new());
16816        let error = context
16817            .append_workflow_stream(
16818                "output",
16819                &[WorkflowStreamAppendItem::from_reference(
16820                    "s3://bucket/item.avro",
16821                )],
16822                None,
16823            )
16824            .expect_err("stream append without durable command identity must fail closed");
16825
16826        assert!(matches!(error, Error::MissingWorkflowCommandIdentity));
16827        assert!(context.take_commands().expect("commands").is_empty());
16828    }
16829
16830    #[test]
16831    fn cold_worker_replay_does_not_repeat_committed_side_effects_or_markers() {
16832        fn worker(calls: Arc<AtomicUsize>) -> Worker {
16833            let client = Client::new("http://127.0.0.1:8080").expect("client");
16834            let mut worker = Worker::new(client, "rust-workers");
16835            worker.register_workflow("rust.side-effect-version", move |ctx, _input| {
16836                let calls = Arc::clone(&calls);
16837                async move {
16838                    let captured = ctx.side_effect(|| {
16839                        calls.fetch_add(1, Ordering::SeqCst);
16840                        "captured-once".to_string()
16841                    })?;
16842                    let version = ctx.get_version("cold-restart", 1, 2)?;
16843                    Ok(json!({"captured": captured, "version": version}))
16844                }
16845            });
16846            worker
16847        }
16848
16849        fn task(history_events: Vec<HistoryEvent>) -> WorkflowTask {
16850            WorkflowTask {
16851                task_id: "wft-side-effect-version".to_string(),
16852                workflow_command_id: None,
16853                workflow_id: Some("wf-side-effect-version".to_string()),
16854                run_id: Some("run-side-effect-version".to_string()),
16855                workflow_type: "rust.side-effect-version".to_string(),
16856                cancel_requested: false,
16857                payload_codec: DEFAULT_CODEC.to_string(),
16858                arguments: Some(
16859                    encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("arguments"),
16860                ),
16861                history_events,
16862                total_history_events: None,
16863                history_size_bytes: None,
16864                continue_as_new_recommended: None,
16865                history_budget_pressure: None,
16866                next_history_page_token: None,
16867                workflow_task_attempt: 1,
16868                workflow_signal_id: None,
16869                signal_name: None,
16870                signal_arguments: None,
16871                workflow_update_id: None,
16872                update_name: None,
16873                lease_owner: Some("rust-worker".to_string()),
16874            }
16875        }
16876
16877        let calls = Arc::new(AtomicUsize::new(0));
16878        let initial = worker(Arc::clone(&calls))
16879            .execute_workflow_task(task(Vec::new()))
16880            .expect("initial execution");
16881        assert_eq!(
16882            initial
16883                .iter()
16884                .map(|command| &command["type"])
16885                .collect::<Vec<_>>(),
16886            vec![
16887                "record_side_effect",
16888                "record_version_marker",
16889                "complete_workflow"
16890            ]
16891        );
16892        assert_eq!(calls.load(Ordering::SeqCst), 1);
16893
16894        let restarted = worker(Arc::clone(&calls));
16895        let replayed = restarted
16896            .execute_workflow_task(task(vec![
16897                history_event(
16898                    "SideEffectRecorded",
16899                    json!({"sequence": 1, "result": initial[0]["result"].clone()}),
16900                ),
16901                history_event(
16902                    "VersionMarkerRecorded",
16903                    json!({
16904                        "sequence": 2,
16905                        "change_id": "cold-restart",
16906                        "version": 2,
16907                        "min_supported": 1,
16908                        "max_supported": 2,
16909                    }),
16910                ),
16911            ]))
16912            .expect("cold replay");
16913        assert_eq!(replayed.len(), 1);
16914        assert_eq!(replayed[0]["type"], "complete_workflow");
16915        assert_eq!(calls.load(Ordering::SeqCst), 1);
16916    }
16917
16918    #[test]
16919    fn side_effect_replay_rejects_changed_rust_value_type() {
16920        let result = encode_value_envelope(&json!({"value": 42}), DEFAULT_CODEC).expect("result");
16921        let ctx = workflow_context(vec![history_event(
16922            "SideEffectRecorded",
16923            json!({"sequence": 1, "result": result}),
16924        )]);
16925        let error = ctx
16926            .side_effect::<Vec<String>, _>(|| panic!("callback must not run"))
16927            .expect_err("changed type must fail replay");
16928        assert!(matches!(
16929            error,
16930            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
16931                if reason == "side_effect_type_mismatch"
16932        ));
16933    }
16934
16935    fn completed_retry_activity_history() -> Vec<HistoryEvent> {
16936        vec![
16937            history_event(
16938                "ActivityScheduled",
16939                json!({
16940                    "sequence": 1,
16941                    "activity_type": "flaky",
16942                    "activity_execution_id": "act-1",
16943                    "activity": {
16944                        "id": "act-1",
16945                        "sequence": 1,
16946                        "type": "flaky",
16947                        "queue": "critical-activities",
16948                        "execution_mode": null,
16949                        "retry_policy": {
16950                            "snapshot_version": 1,
16951                            "max_attempts": 3,
16952                            "backoff_seconds": [2, 4],
16953                            "start_to_close_timeout": 30,
16954                            "schedule_to_start_timeout": 5,
16955                            "schedule_to_close_timeout": 90,
16956                            "heartbeat_timeout": 10,
16957                            "non_retryable_error_types": ["PermanentError"]
16958                        }
16959                    }
16960                }),
16961            ),
16962            history_event(
16963                "ActivityStarted",
16964                json!({
16965                    "sequence": 1,
16966                    "activity_type": "flaky",
16967                    "activity_execution_id": "act-1",
16968                    "activity_attempt_id": "attempt-1",
16969                    "attempt_number": 1
16970                }),
16971            ),
16972            history_event(
16973                "ActivityRetryScheduled",
16974                json!({
16975                    "sequence": 1,
16976                    "activity_type": "flaky",
16977                    "activity_execution_id": "act-1",
16978                    "activity_attempt_id": "attempt-1",
16979                    "attempt_number": 1,
16980                    "retry_after_attempt": 1,
16981                    "retry_backoff_seconds": 2,
16982                    "failure_category": "activity",
16983                    "exception_type": "TransientError"
16984                }),
16985            ),
16986            history_event(
16987                "ActivityStarted",
16988                json!({
16989                    "sequence": 1,
16990                    "activity_type": "flaky",
16991                    "activity_execution_id": "act-1",
16992                    "activity_attempt_id": "attempt-2",
16993                    "attempt_number": 2
16994                }),
16995            ),
16996            history_event(
16997                "ActivityCompleted",
16998                json!({
16999                    "sequence": 1,
17000                    "activity_type": "flaky",
17001                    "activity_execution_id": "act-1",
17002                    "activity_attempt_id": "attempt-2",
17003                    "attempt_number": 2,
17004                    "payload_codec": DEFAULT_CODEC,
17005                    "result": fixture_envelope(json!({"status":"recovered"}))
17006                }),
17007            ),
17008        ]
17009    }
17010
17011    fn retry_activity_options() -> ActivityOptions {
17012        ActivityOptions::new()
17013            .task_queue("critical-activities")
17014            .retry_policy(
17015                ActivityRetryPolicy::new(3)
17016                    .backoff_intervals([Duration::from_secs(2), Duration::from_secs(4)])
17017                    .non_retryable_error_type("PermanentError"),
17018            )
17019            .start_to_close_timeout(Duration::from_secs(30))
17020            .schedule_to_start_timeout(Duration::from_secs(5))
17021            .schedule_to_close_timeout(Duration::from_secs(90))
17022            .heartbeat_timeout(Duration::from_secs(10))
17023    }
17024
17025    #[test]
17026    fn fixed_avro_value_round_trips_json_values() {
17027        let value = json!({"greeting": "hello", "count": 3, "ok": true});
17028        let envelope = PayloadEnvelope::avro(&value).expect("encode");
17029        assert_eq!(envelope.codec, DEFAULT_CODEC);
17030        assert_eq!(decode_payload::<Value>(&envelope).expect("decode"), value);
17031    }
17032
17033    #[tokio::test]
17034    async fn typed_handler_adapters_round_trip_serde_contracts_on_the_fixed_wire() {
17035        let client = Client::new("http://127.0.0.1:8080").expect("client");
17036        let mut worker = Worker::new(client, "rust-workers");
17037        worker.register_typed_workflow(
17038            "typed.contract.workflow",
17039            |_ctx, input: TypedContract| async move { Ok(input) },
17040        );
17041        worker.register_typed_activity(
17042            "typed.contract.activity",
17043            |_ctx, input: TypedContract| async move { Ok(input) },
17044        );
17045
17046        let expected = typed_contract();
17047        let arguments = AvroValue::Array(vec![
17048            AvroValue::from_serialize(&expected).expect("typed request")
17049        ]);
17050        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("arguments");
17051        let mut workflow = workflow_task("typed.contract.workflow", Vec::new(), DEFAULT_CODEC);
17052        workflow.arguments = Some(envelope.clone());
17053        let commands = worker
17054            .execute_workflow_task(workflow)
17055            .expect("typed workflow task");
17056        let workflow_result: TypedContract =
17057            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17058                .expect("workflow result envelope")
17059                .deserialize()
17060                .expect("workflow result type");
17061        assert_eq!(workflow_result, expected);
17062
17063        let activity = ActivityTask {
17064            task_id: "typed-contract-activity".to_string(),
17065            activity_attempt_id: Some("typed-contract-attempt".to_string()),
17066            attempt_id: None,
17067            activity_type: "typed.contract.activity".to_string(),
17068            payload_codec: DEFAULT_CODEC.to_string(),
17069            arguments: Some(envelope),
17070            attempt_number: 1,
17071            lease_owner: Some("rust-worker".to_string()),
17072        };
17073        let activity_result: TypedContract = worker
17074            .execute_activity_task(activity)
17075            .await
17076            .expect("typed activity task")
17077            .deserialize()
17078            .expect("activity result type");
17079        assert_eq!(activity_result, expected);
17080    }
17081
17082    #[tokio::test]
17083    async fn typed_handler_errors_include_handler_name_direction_and_rust_type() {
17084        let client = Client::new("http://127.0.0.1:8080").expect("client");
17085        let mut worker = Worker::new(client, "rust-workers");
17086        worker.register_typed_workflow(
17087            "typed.shape.workflow",
17088            |_ctx, input: TypedContract| async move { Ok(input) },
17089        );
17090        worker.register_typed_activity("typed.unsupported.activity", |_ctx, (): ()| async move {
17091            Ok(f64::NAN)
17092        });
17093
17094        let mut workflow = workflow_task("typed.shape.workflow", Vec::new(), DEFAULT_CODEC);
17095        workflow.arguments = Some(
17096            encode_typed_envelope(
17097                &AvroValue::Array(vec![
17098                    AvroValue::String("first".to_string()),
17099                    AvroValue::String("second".to_string()),
17100                ]),
17101                DEFAULT_CODEC,
17102            )
17103            .expect("malformed typed arguments"),
17104        );
17105        let commands = worker
17106            .execute_workflow_task(workflow)
17107            .expect("shape mismatch becomes a workflow failure");
17108        let message = commands[0]["message"].as_str().expect("failure message");
17109        assert!(message.contains("workflow handler \"typed.shape.workflow\" input type"));
17110        assert!(message.contains(type_name::<TypedContract>()));
17111        assert!(message.contains("task carried 2 arguments"));
17112
17113        let activity = ActivityTask {
17114            task_id: "typed-unsupported-activity".to_string(),
17115            activity_attempt_id: Some("typed-unsupported-attempt".to_string()),
17116            attempt_id: None,
17117            activity_type: "typed.unsupported.activity".to_string(),
17118            payload_codec: DEFAULT_CODEC.to_string(),
17119            arguments: Some(
17120                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17121                    .expect("unit arguments"),
17122            ),
17123            attempt_number: 1,
17124            lease_owner: Some("rust-worker".to_string()),
17125        };
17126        let Error::HandlerType {
17127            handler_kind,
17128            handler_name,
17129            value_kind,
17130            rust_type,
17131            message,
17132        } = worker
17133            .execute_activity_task(activity)
17134            .await
17135            .expect_err("non-finite handler output must fail")
17136        else {
17137            panic!("expected contextual handler type failure");
17138        };
17139        assert_eq!(handler_kind, HandlerKind::Activity);
17140        assert_eq!(handler_name, "typed.unsupported.activity");
17141        assert_eq!(value_kind, HandlerValueKind::Result);
17142        assert_eq!(rust_type, type_name::<f64>());
17143        assert!(message.contains("non_finite_float"));
17144    }
17145
17146    #[tokio::test]
17147    async fn typed_replayed_workflow_decodes_input_and_activity_result_losslessly() {
17148        #[derive(Clone, Default)]
17149        struct State {
17150            observed: Option<TypedContract>,
17151        }
17152
17153        let client = Client::new("http://127.0.0.1:8080").expect("client");
17154        let mut worker = Worker::new(client, "rust-workers");
17155        worker.register_typed_replayed_workflow(
17156            "typed.contract.replayed",
17157            State::default,
17158            |ctx, input: TypedContract, state| async move {
17159                let result: TypedContract =
17160                    ctx.activity_typed("typed.contract.activity", input).await?;
17161                state.update(|current| current.observed = Some(result.clone()))?;
17162                Ok(result)
17163            },
17164        );
17165        worker.register_replayed_query::<State, _, _>(
17166            "typed.contract.replayed",
17167            "observed",
17168            |_ctx, state, _args| async move {
17169                Ok(json!(state.observed.as_ref().map(|value| value.signed)))
17170            },
17171        );
17172
17173        let expected = typed_contract();
17174        let typed_value = AvroValue::from_serialize(&expected).expect("typed value");
17175        let workflow_arguments =
17176            encode_typed_envelope(&AvroValue::Array(vec![typed_value.clone()]), DEFAULT_CODEC)
17177                .expect("workflow arguments");
17178        let result = encode_typed_envelope(&typed_value, DEFAULT_CODEC).expect("activity result");
17179        let task = QueryTask {
17180            query_task_id: "typed-replay-query".to_string(),
17181            query_task_attempt: 1,
17182            lease_owner: Some("rust-worker".to_string()),
17183            workflow_id: Some("typed-replay".to_string()),
17184            run_id: Some("typed-replay-run".to_string()),
17185            workflow_type: "typed.contract.replayed".to_string(),
17186            query_name: "observed".to_string(),
17187            payload_codec: DEFAULT_CODEC.to_string(),
17188            workflow_arguments: Some(workflow_arguments),
17189            query_arguments: Some(
17190                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17191                    .expect("query arguments"),
17192            ),
17193            history_events: vec![
17194                history_event(
17195                    "ActivityScheduled",
17196                    json!({
17197                        "sequence": 1,
17198                        "activity_type": "typed.contract.activity"
17199                    }),
17200                ),
17201                history_event(
17202                    "ActivityCompleted",
17203                    json!({
17204                        "sequence": 1,
17205                        "activity_type": "typed.contract.activity",
17206                        "payload_codec": DEFAULT_CODEC,
17207                        "result": result
17208                    }),
17209                ),
17210            ],
17211            history_export: None,
17212            run_status: Some("completed".to_string()),
17213        };
17214
17215        assert_eq!(
17216            worker
17217                .execute_query_task(task)
17218                .await
17219                .expect("typed replay query")
17220                .deserialize::<i64>()
17221                .expect("query result"),
17222            expected.signed
17223        );
17224    }
17225
17226    #[tokio::test]
17227    async fn typed_worker_surfaces_preserve_bytes_and_map_list_identity() {
17228        let client = Client::new("http://127.0.0.1:8080").expect("client");
17229        let mut worker = Worker::new(client, "rust-workers");
17230        worker.register_workflow_avro_value("typed.echo", |_ctx, input| async move { Ok(input) });
17231        worker
17232            .register_activity_avro_value("typed.activity", |_ctx, input| async move { Ok(input) });
17233        worker.register_query_avro_value("typed.echo", "inspect", |_ctx, input| async move {
17234            Ok(input)
17235        });
17236        worker.register_update_avro_value("typed.echo", "replace", |_ctx, input| async move {
17237            Ok(input)
17238        });
17239        worker.register_workflow_avro_value("typed.signal", |ctx, _input| async move {
17240            Ok(AvroValue::Array(
17241                ctx.wait_signal_avro_value("changed").await?,
17242            ))
17243        });
17244
17245        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
17246        let envelope = encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed envelope");
17247
17248        let mut workflow = workflow_task("typed.echo", Vec::new(), DEFAULT_CODEC);
17249        workflow.arguments = Some(envelope.clone());
17250        let commands = worker
17251            .execute_workflow_task(workflow)
17252            .expect("typed workflow task");
17253        assert_eq!(commands[0]["type"], "complete_workflow");
17254        assert_eq!(
17255            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17256                .expect("typed workflow result"),
17257            arguments
17258        );
17259
17260        let activity = ActivityTask {
17261            task_id: "activity-typed".to_string(),
17262            activity_attempt_id: Some("attempt-typed".to_string()),
17263            attempt_id: None,
17264            activity_type: "typed.activity".to_string(),
17265            payload_codec: DEFAULT_CODEC.to_string(),
17266            arguments: Some(envelope.clone()),
17267            attempt_number: 1,
17268            lease_owner: Some("rust-worker".to_string()),
17269        };
17270        assert_eq!(
17271            worker
17272                .execute_activity_task(activity)
17273                .await
17274                .expect("typed activity result"),
17275            arguments
17276        );
17277
17278        let query = QueryTask {
17279            query_task_id: "query-typed".to_string(),
17280            query_task_attempt: 1,
17281            lease_owner: Some("rust-worker".to_string()),
17282            workflow_id: Some("typed-1".to_string()),
17283            run_id: Some("run-typed".to_string()),
17284            workflow_type: "typed.echo".to_string(),
17285            query_name: "inspect".to_string(),
17286            payload_codec: DEFAULT_CODEC.to_string(),
17287            workflow_arguments: Some(
17288                encode_typed_envelope(&AvroValue::Array(Vec::new()), DEFAULT_CODEC)
17289                    .expect("workflow input"),
17290            ),
17291            query_arguments: Some(envelope.clone()),
17292            history_events: Vec::new(),
17293            history_export: None,
17294            run_status: Some("running".to_string()),
17295        };
17296        assert_eq!(
17297            worker
17298                .execute_query_task(query)
17299                .await
17300                .expect("typed query result"),
17301            arguments
17302        );
17303
17304        let mut update = workflow_task(
17305            "typed.echo",
17306            vec![history_event(
17307                "UpdateAccepted",
17308                json!({
17309                    "update_id": "update-typed",
17310                    "update_name": "replace",
17311                    "arguments": envelope.clone(),
17312                }),
17313            )],
17314            DEFAULT_CODEC,
17315        );
17316        update.workflow_update_id = Some("update-typed".to_string());
17317        update.update_name = Some("replace".to_string());
17318        let commands = worker
17319            .execute_workflow_task(update)
17320            .expect("typed update task");
17321        assert_eq!(commands[0]["type"], "complete_update");
17322        assert_eq!(
17323            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17324                .expect("typed update result"),
17325            arguments
17326        );
17327
17328        let mut signal = workflow_task(
17329            "typed.signal",
17330            vec![history_event(
17331                "SignalReceived",
17332                json!({
17333                    "signal_id": "signal-typed",
17334                    "signal_name": "changed",
17335                    "arguments": envelope.clone(),
17336                }),
17337            )],
17338            DEFAULT_CODEC,
17339        );
17340        signal.workflow_signal_id = Some("signal-typed".to_string());
17341        signal.signal_name = Some("changed".to_string());
17342        signal.signal_arguments = Some(envelope);
17343        let commands = worker
17344            .execute_workflow_task(signal)
17345            .expect("typed signal resume");
17346        assert_eq!(
17347            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
17348                .expect("typed signal result"),
17349            arguments
17350        );
17351    }
17352
17353    #[tokio::test]
17354    async fn typed_helpers_never_parse_json_inspection_projection() {
17355        let collision_values = projection_collision_probe();
17356        let expected = AvroValue::Array(collision_values.clone());
17357        let envelope = encode_typed_envelope(&expected, DEFAULT_CODEC).expect("collision envelope");
17358
17359        let activity_context = workflow_context_with_codec(
17360            vec![history_event(
17361                "ActivityCompleted",
17362                json!({
17363                    "sequence": 1,
17364                    "activity_type": "collision.activity",
17365                    "payload_codec": DEFAULT_CODEC,
17366                    "result": envelope.clone(),
17367                }),
17368            )],
17369            DEFAULT_CODEC,
17370        );
17371        assert_eq!(
17372            activity_context
17373                .activity_avro_value("collision.activity", AvroValue::Array(Vec::new()))
17374                .await
17375                .expect("typed activity collision result"),
17376            expected
17377        );
17378
17379        let signal_context = workflow_context_with_codec(
17380            vec![
17381                history_event(
17382                    "SignalWaitOpened",
17383                    json!({"sequence": 1, "signal_name": "collision"}),
17384                ),
17385                history_event(
17386                    "SignalApplied",
17387                    json!({
17388                        "sequence": 1,
17389                        "signal_name": "collision",
17390                        "payload_codec": DEFAULT_CODEC,
17391                        "value": envelope.clone(),
17392                    }),
17393                ),
17394            ],
17395            DEFAULT_CODEC,
17396        );
17397        assert_eq!(
17398            signal_context
17399                .wait_signal_avro_value("collision")
17400                .await
17401                .expect("typed signal collision arguments"),
17402            collision_values
17403        );
17404
17405        let child_context = workflow_context_with_codec(
17406            vec![
17407                history_event(
17408                    "ChildWorkflowScheduled",
17409                    json!({
17410                        "sequence": 1,
17411                        "child_workflow_instance_id": "collision-child",
17412                        "child_workflow_run_id": "collision-run",
17413                        "child_workflow_type": "collision.child",
17414                    }),
17415                ),
17416                history_event(
17417                    "ChildRunCompleted",
17418                    json!({
17419                        "sequence": 1,
17420                        "child_workflow_instance_id": "collision-child",
17421                        "child_workflow_run_id": "collision-run",
17422                        "child_workflow_type": "collision.child",
17423                        "payload_codec": DEFAULT_CODEC,
17424                        "result": envelope,
17425                    }),
17426                ),
17427            ],
17428            DEFAULT_CODEC,
17429        );
17430        let child = child_context
17431            .start_child_workflow_avro_value(
17432                "collision.child",
17433                ChildWorkflowOptions::new("collision-workers"),
17434                AvroValue::Array(Vec::new()),
17435            )
17436            .await
17437            .expect("typed child collision result");
17438        assert_eq!(child.result, expected);
17439    }
17440
17441    #[tokio::test]
17442    async fn replayed_typed_query_keeps_lossless_workflow_and_query_inputs() {
17443        let client = Client::new("http://127.0.0.1:8080").expect("client");
17444        let mut worker = Worker::new(client, "rust-workers");
17445        worker.register_replayed_workflow_avro_value(
17446            "typed.replayed",
17447            || (),
17448            |_ctx, input, _state| async move { Ok(input) },
17449        );
17450        worker.register_replayed_query_avro_value::<(), _, _>(
17451            "typed.replayed",
17452            "inspect",
17453            |ctx, _state, args| async move {
17454                let mut signals = ctx.signals_avro_value("collision");
17455                let signal = signals
17456                    .pop()
17457                    .map(AvroValue::Array)
17458                    .unwrap_or_else(|| AvroValue::Array(Vec::new()));
17459                Ok(AvroValue::Array(vec![
17460                    ctx.workflow_input_avro_value().clone(),
17461                    signal,
17462                    args,
17463                ]))
17464            },
17465        );
17466        let arguments = AvroValue::Array(projection_collision_probe());
17467        let signal_arguments =
17468            encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("typed query signal arguments");
17469        let task = QueryTask {
17470            query_task_id: "query-typed-replay".to_string(),
17471            query_task_attempt: 1,
17472            lease_owner: Some("rust-worker".to_string()),
17473            workflow_id: Some("typed-replay".to_string()),
17474            run_id: Some("run-typed-replay".to_string()),
17475            workflow_type: "typed.replayed".to_string(),
17476            query_name: "inspect".to_string(),
17477            payload_codec: DEFAULT_CODEC.to_string(),
17478            workflow_arguments: Some(
17479                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("workflow arguments"),
17480            ),
17481            query_arguments: Some(
17482                encode_typed_envelope(&arguments, DEFAULT_CODEC).expect("query arguments"),
17483            ),
17484            history_events: vec![history_event(
17485                "SignalReceived",
17486                json!({
17487                    "signal_id": "collision-signal",
17488                    "signal_name": "collision",
17489                    "workflow_sequence": 1,
17490                    "payload_codec": DEFAULT_CODEC,
17491                    "arguments": signal_arguments,
17492                }),
17493            )],
17494            history_export: None,
17495            run_status: Some("completed".to_string()),
17496        };
17497
17498        assert_eq!(
17499            worker
17500                .execute_query_task(task)
17501                .await
17502                .expect("typed replay query"),
17503            AvroValue::Array(vec![arguments.clone(), arguments.clone(), arguments])
17504        );
17505    }
17506
17507    #[test]
17508    fn public_avro_adapter_rejects_non_string_map_keys_before_json_conversion() {
17509        let value = BTreeMap::from([(1_i32, "integer key")]);
17510        let error = PayloadEnvelope::avro(&value)
17511            .expect_err("integer map keys must fail")
17512            .to_string();
17513
17514        assert!(error.contains("invalid_map_key"));
17515    }
17516
17517    #[test]
17518    fn json_tagged_payload_fails_closed_with_actionable_diagnostic() {
17519        let envelope = PayloadEnvelope {
17520            codec: "json".to_string(),
17521            blob: r#"{"greeting":"hello"}"#.to_string(),
17522        };
17523
17524        let error = decode_payload::<Value>(&envelope).expect_err("JSON payload must fail");
17525        let diagnostic = error.to_string();
17526        assert!(diagnostic.contains("unsupported_payload_codec"));
17527        assert!(diagnostic.contains("codec=\"avro\""));
17528        assert!(diagnostic.contains("HTTP document transport"));
17529    }
17530
17531    #[test]
17532    fn untagged_json_payload_value_fails_closed() {
17533        let error = decode_wire_value(&json!({"stale": true}), DEFAULT_CODEC)
17534            .expect_err("untagged JSON payload values must fail");
17535        let diagnostic = error.to_string();
17536        assert!(diagnostic.contains("unsupported_payload_codec"));
17537        assert!(diagnostic.contains("untagged durable payload"));
17538        assert!(diagnostic.contains("HTTP document transport"));
17539    }
17540
17541    #[test]
17542    fn prerelease_avro_payload_without_single_object_frame_is_rejected() {
17543        let envelope = PayloadEnvelope {
17544            codec: DEFAULT_CODEC.to_string(),
17545            blob: BASE64.encode([0x01]),
17546        };
17547
17548        let error = decode_payload::<Value>(&envelope).expect_err("prerelease payload must fail");
17549        assert!(error.to_string().contains("invalid_payload_framing"));
17550    }
17551
17552    #[tokio::test]
17553    async fn workflow_completion_rejects_invalid_payload_slots_without_transport() {
17554        let server = MockWorkerServer::start();
17555        let client = Client::builder(server.base_url())
17556            .timeout(Duration::from_secs(2))
17557            .build()
17558            .expect("client");
17559        let invalid_commands = [
17560            json!({
17561                "type": "complete_workflow",
17562                "result": {"codec": "json", "blob": null}
17563            }),
17564            json!({
17565                "type": "schedule_activity",
17566                "arguments": {"codec": "yaml", "blob": "ignored"}
17567            }),
17568            json!({
17569                "type": "start_child_workflow",
17570                "arguments": {"codec": DEFAULT_CODEC, "blob": null}
17571            }),
17572            json!({"type": "continue_as_new", "arguments": []}),
17573            json!({"type": "complete_update"}),
17574            json!({"type": "record_side_effect", "result": null}),
17575            json!({
17576                "type": "start_service_operation",
17577                "payload_codec": DEFAULT_CODEC,
17578                "request_payload": "raw-avro-bytes"
17579            }),
17580        ];
17581
17582        for command in invalid_commands {
17583            let error = client
17584                .complete_workflow_task("invalid-codec", "rust-worker", 1, vec![command])
17585                .await
17586                .expect_err("invalid durable payload must fail locally");
17587            let diagnostic = error.to_string();
17588            assert!(
17589                diagnostic.contains("unsupported_payload_codec")
17590                    || diagnostic.contains("invalid_payload_envelope")
17591                    || diagnostic.contains("untagged durable payload"),
17592                "unexpected validation diagnostic: {diagnostic}"
17593            );
17594        }
17595
17596        assert_eq!(
17597            server.request_count("/api/worker/workflow-tasks/invalid-codec/complete"),
17598            0,
17599            "invalid command payloads must not reach HTTP transport"
17600        );
17601    }
17602
17603    #[test]
17604    fn workflow_completion_validates_only_protocol_owned_payload_slots() {
17605        let envelope = fixture_envelope(json!({"codec": "customer-value"}));
17606        let commands = [
17607            json!({"type": "complete_workflow", "result": envelope.clone()}),
17608            json!({"type": "schedule_activity", "arguments": envelope.clone()}),
17609            json!({"type": "start_child_workflow", "arguments": envelope.clone()}),
17610            json!({"type": "continue_as_new", "arguments": envelope.clone()}),
17611            json!({"type": "complete_update", "result": envelope.clone()}),
17612            json!({"type": "record_side_effect", "result": envelope.clone()}),
17613            json!({
17614                "type": "start_service_operation",
17615                "payload_codec": DEFAULT_CODEC,
17616                "request_payload": envelope.clone()
17617            }),
17618            json!({
17619                "type": "complete_workflow",
17620                "result": envelope,
17621                "metadata": {
17622                    "codec": "json",
17623                    "payload_codec": "customer-codec",
17624                    "result": {"codec": "yaml", "blob": null}
17625                }
17626            }),
17627        ];
17628
17629        validate_workflow_task_commands(&commands)
17630            .expect("customer metadata must not become a protocol codec declaration");
17631    }
17632
17633    #[test]
17634    fn valid_avro_tasks_normalize_absent_and_null_arguments_to_empty_lists() {
17635        assert_eq!(
17636            decode_task_avro_arguments(None, DEFAULT_CODEC).expect("absent arguments"),
17637            AvroValue::Array(Vec::new())
17638        );
17639        assert_eq!(
17640            decode_task_avro_arguments(Some(&Value::Null), DEFAULT_CODEC).expect("null arguments"),
17641            AvroValue::Array(Vec::new())
17642        );
17643
17644        let mut signal = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17645        signal.signal_name = Some("empty-signal".to_string());
17646        signal.signal_arguments = None;
17647        let decoded = decode_resume_signal(&signal)
17648            .expect("valid Avro signal")
17649            .expect("named signal resumes the workflow");
17650        assert!(decoded.arguments.is_empty());
17651    }
17652
17653    #[tokio::test]
17654    async fn malformed_task_level_codecs_become_pre_handler_failures() {
17655        let client = Client::new("http://127.0.0.1:8080").expect("client");
17656        let mut worker = Worker::new(client, "rust-workers");
17657        let handler_calls = Arc::new(AtomicUsize::new(0));
17658
17659        let calls = Arc::clone(&handler_calls);
17660        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17661            calls.fetch_add(1, Ordering::SeqCst);
17662            async move { Ok(Value::Null) }
17663        });
17664        let calls = Arc::clone(&handler_calls);
17665        worker.register_activity("codec.activity", move |_ctx, _args| {
17666            calls.fetch_add(1, Ordering::SeqCst);
17667            async move { Ok(Value::Null) }
17668        });
17669        let calls = Arc::clone(&handler_calls);
17670        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17671            calls.fetch_add(1, Ordering::SeqCst);
17672            async move { Ok(Value::Null) }
17673        });
17674
17675        let mut failures = Vec::new();
17676        for codec_case in [
17677            InvalidTaskPayloadCodec::Missing,
17678            InvalidTaskPayloadCodec::Null,
17679            InvalidTaskPayloadCodec::NonString,
17680        ] {
17681            let mut workflow = json!({
17682                "task_id": format!("workflow-{}", codec_case.label()),
17683                "workflow_type": "codec.workflow"
17684            });
17685            codec_case.apply(&mut workflow);
17686            match serde_json::from_value::<WorkflowTask>(workflow) {
17687                Ok(task) => match worker.execute_workflow_task(task) {
17688                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17689                    outcome => failures.push(format!(
17690                        "workflow {} codec returned {outcome:?}",
17691                        codec_case.label()
17692                    )),
17693                },
17694                Err(error) => failures.push(format!(
17695                    "workflow {} codec failed transport deserialization: {error}",
17696                    codec_case.label()
17697                )),
17698            }
17699
17700            let mut activity = json!({
17701                "task_id": format!("activity-{}", codec_case.label()),
17702                "activity_attempt_id": format!("attempt-{}", codec_case.label()),
17703                "activity_type": "codec.activity",
17704                "attempt_number": 1
17705            });
17706            codec_case.apply(&mut activity);
17707            match serde_json::from_value::<ActivityTask>(activity) {
17708                Ok(task) => match worker.execute_activity_task(task).await {
17709                    Err(error) if error.to_string().contains("unsupported_payload_codec") => {}
17710                    outcome => failures.push(format!(
17711                        "activity {} codec returned {outcome:?}",
17712                        codec_case.label()
17713                    )),
17714                },
17715                Err(error) => failures.push(format!(
17716                    "activity {} codec failed transport deserialization: {error}",
17717                    codec_case.label()
17718                )),
17719            }
17720
17721            let mut query = json!({
17722                "query_task_id": format!("query-{}", codec_case.label()),
17723                "workflow_type": "codec.workflow",
17724                "query_name": "known"
17725            });
17726            codec_case.apply(&mut query);
17727            match serde_json::from_value::<QueryTask>(query) {
17728                Ok(task) => match worker.execute_query_task(task).await {
17729                    Err(failure) if failure.message.contains("unsupported_payload_codec") => {}
17730                    outcome => failures.push(format!(
17731                        "query {} codec returned {outcome:?}",
17732                        codec_case.label()
17733                    )),
17734                },
17735                Err(error) => failures.push(format!(
17736                    "query {} codec failed transport deserialization: {error}",
17737                    codec_case.label()
17738                )),
17739            }
17740        }
17741
17742        assert!(failures.is_empty(), "{}", failures.join("\n"));
17743        assert_eq!(
17744            handler_calls.load(Ordering::SeqCst),
17745            0,
17746            "invalid task codecs must not invoke a handler"
17747        );
17748    }
17749
17750    #[tokio::test]
17751    async fn polled_malformed_task_codecs_are_settled_without_handler_execution() {
17752        for codec_case in [
17753            InvalidTaskPayloadCodec::Missing,
17754            InvalidTaskPayloadCodec::Null,
17755            InvalidTaskPayloadCodec::NonString,
17756        ] {
17757            let server = MockWorkerServer::invalid_task_payload_codec(codec_case);
17758            let client = Client::builder(server.base_url())
17759                .timeout(Duration::from_secs(2))
17760                .build()
17761                .expect("client");
17762            let mut worker = Worker::new(client, "rust-workers")
17763                .worker_id("codec-worker")
17764                .poll_timeout(Duration::from_millis(10));
17765            let handler_calls = Arc::new(AtomicUsize::new(0));
17766
17767            let calls = Arc::clone(&handler_calls);
17768            worker.register_workflow("codec.workflow", move |_ctx, _args| {
17769                calls.fetch_add(1, Ordering::SeqCst);
17770                async move { Ok(Value::Null) }
17771            });
17772            let calls = Arc::clone(&handler_calls);
17773            worker.register_activity("codec.activity", move |_ctx, _args| {
17774                calls.fetch_add(1, Ordering::SeqCst);
17775                async move { Ok(Value::Null) }
17776            });
17777            let calls = Arc::clone(&handler_calls);
17778            worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17779                calls.fetch_add(1, Ordering::SeqCst);
17780                async move { Ok(Value::Null) }
17781            });
17782
17783            assert_eq!(
17784                worker.run_once().await.expect("invalid tasks are settled"),
17785                3,
17786                "all {} codec tasks must be handled",
17787                codec_case.label()
17788            );
17789            assert_eq!(
17790                handler_calls.load(Ordering::SeqCst),
17791                0,
17792                "{} task codecs must fail before every handler",
17793                codec_case.label()
17794            );
17795
17796            for path in [
17797                "/api/worker/workflow-tasks/codec-workflow/fail",
17798                "/api/worker/activity-tasks/codec-activity/fail",
17799                "/api/worker/query-tasks/codec-query/fail",
17800            ] {
17801                let body = server.request_body(path);
17802                assert!(
17803                    body["failure"]["message"]
17804                        .as_str()
17805                        .is_some_and(|message| message.contains("unsupported_payload_codec")),
17806                    "{path} must receive the stable codec diagnostic for the {} case: {body}",
17807                    codec_case.label()
17808                );
17809            }
17810            assert_eq!(
17811                server.request_body("/api/worker/query-tasks/codec-query/fail")["failure"]
17812                    ["reason"],
17813                "query_payload_decode_failed"
17814            );
17815            for path in [
17816                "/api/worker/workflow-tasks/codec-workflow/complete",
17817                "/api/worker/activity-tasks/codec-activity/complete",
17818                "/api/worker/query-tasks/codec-query/complete",
17819            ] {
17820                assert_eq!(
17821                    server.request_count(path),
17822                    0,
17823                    "invalid {} codec task reached {path}",
17824                    codec_case.label()
17825                );
17826            }
17827        }
17828    }
17829
17830    #[tokio::test]
17831    async fn invalid_inbound_codecs_precede_handlers_and_unrelated_outcomes() {
17832        let client = Client::new("http://127.0.0.1:8080").expect("client");
17833        let mut worker = Worker::new(client, "rust-workers");
17834        let handler_calls = Arc::new(AtomicUsize::new(0));
17835
17836        let calls = Arc::clone(&handler_calls);
17837        worker.register_workflow("codec.workflow", move |_ctx, _args| {
17838            calls.fetch_add(1, Ordering::SeqCst);
17839            async move { Ok(Value::Null) }
17840        });
17841        let calls = Arc::clone(&handler_calls);
17842        worker.register_activity("codec.activity", move |_ctx, _args| {
17843            calls.fetch_add(1, Ordering::SeqCst);
17844            async move { Ok(Value::Null) }
17845        });
17846        let calls = Arc::clone(&handler_calls);
17847        worker.register_update("codec.workflow", "known", move |_ctx, _args| {
17848            calls.fetch_add(1, Ordering::SeqCst);
17849            async move { Ok(Value::Null) }
17850        });
17851        let calls = Arc::clone(&handler_calls);
17852        worker.register_query("codec.workflow", "known", move |_ctx, _args| {
17853            calls.fetch_add(1, Ordering::SeqCst);
17854            async move { Ok(Value::Null) }
17855        });
17856
17857        let mut workflow = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17858        workflow.payload_codec = "json".to_string();
17859        workflow.arguments = None;
17860        let error = worker
17861            .execute_workflow_task(workflow)
17862            .expect_err("task codec must be checked before workflow invocation");
17863        assert!(error.to_string().contains("unsupported_payload_codec"));
17864
17865        let activity = ActivityTask {
17866            task_id: "activity-invalid-codec".to_string(),
17867            activity_attempt_id: None,
17868            attempt_id: None,
17869            activity_type: "codec.activity".to_string(),
17870            payload_codec: "unknown".to_string(),
17871            arguments: None,
17872            attempt_number: 1,
17873            lease_owner: None,
17874        };
17875        let error = worker
17876            .execute_activity_task(activity)
17877            .await
17878            .expect_err("task codec must be checked before activity invocation");
17879        assert!(error.to_string().contains("unsupported_payload_codec"));
17880
17881        let mut update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17882        update.workflow_update_id = Some("update-invalid-codec".to_string());
17883        update.update_name = Some("known".to_string());
17884        update.history_events.push(history_event(
17885            "UpdateAccepted",
17886            json!({
17887                "update_id": "update-invalid-codec",
17888                "update_name": "known",
17889                "arguments": {"codec": "json", "blob": null}
17890            }),
17891        ));
17892        let error = worker
17893            .execute_workflow_task(update)
17894            .expect_err("nested update codec must be checked before handler lookup");
17895        assert!(error.to_string().contains("unsupported_payload_codec"));
17896
17897        let query: QueryTask = serde_json::from_value(json!({
17898            "query_task_id": "query-invalid-codec",
17899            "workflow_type": "codec.workflow",
17900            "query_name": "known",
17901            "payload_codec": DEFAULT_CODEC,
17902            "workflow_arguments": null,
17903            "query_arguments": null,
17904            "history_export": {
17905                "payloads": {"codec": DEFAULT_CODEC},
17906                "signals": [{
17907                    "name": "empty",
17908                    "payload_codec": "json",
17909                    "arguments": null
17910                }]
17911            }
17912        }))
17913        .expect("query task");
17914        let failure = worker
17915            .execute_query_task(query)
17916            .await
17917            .expect_err("exported signal codec must be checked before query invocation");
17918        assert_eq!(failure.reason, "query_payload_decode_failed");
17919        assert!(failure.message.contains("unsupported_payload_codec"));
17920
17921        let exported_history: QueryTask = serde_json::from_value(json!({
17922            "query_task_id": "query-invalid-history-codec",
17923            "workflow_type": "codec.workflow",
17924            "query_name": "known",
17925            "payload_codec": DEFAULT_CODEC,
17926            "history_export": {
17927                "payloads": {"codec": DEFAULT_CODEC},
17928                "history_events": [{
17929                    "type": "ActivityCompleted",
17930                    "payload": {"payload_codec": "unknown", "result": null}
17931                }]
17932            }
17933        }))
17934        .expect("query task");
17935        let failure = worker
17936            .execute_query_task(exported_history)
17937            .await
17938            .expect_err("exported history codec must be checked before query invocation");
17939        assert_eq!(failure.reason, "query_payload_decode_failed");
17940        assert!(failure.message.contains("unsupported_payload_codec"));
17941        assert_eq!(handler_calls.load(Ordering::SeqCst), 0);
17942
17943        let mut unknown_workflow = workflow_task("missing", Vec::new(), DEFAULT_CODEC);
17944        unknown_workflow.arguments = None;
17945        unknown_workflow.history_events.push(history_event(
17946            "SignalReceived",
17947            json!({
17948                "signal_name": "empty",
17949                "payload_codec": "json",
17950                "arguments": null
17951            }),
17952        ));
17953        let error = worker
17954            .execute_workflow_task(unknown_workflow)
17955            .expect_err("history codec must precede unknown workflow outcome");
17956        assert!(error.to_string().contains("unsupported_payload_codec"));
17957
17958        let unknown_activity = ActivityTask {
17959            task_id: "activity-unknown".to_string(),
17960            activity_attempt_id: None,
17961            attempt_id: None,
17962            activity_type: "missing".to_string(),
17963            payload_codec: "json".to_string(),
17964            arguments: None,
17965            attempt_number: 1,
17966            lease_owner: None,
17967        };
17968        let error = worker
17969            .execute_activity_task(unknown_activity)
17970            .await
17971            .expect_err("codec must precede unknown activity outcome");
17972        assert!(error.to_string().contains("unsupported_payload_codec"));
17973
17974        let mut unknown_update = workflow_task("codec.workflow", Vec::new(), DEFAULT_CODEC);
17975        unknown_update.payload_codec = "json".to_string();
17976        unknown_update.arguments = None;
17977        unknown_update.workflow_update_id = Some("update-unknown".to_string());
17978        unknown_update.update_name = Some("missing".to_string());
17979        let error = worker
17980            .execute_workflow_task(unknown_update)
17981            .expect_err("codec must precede fail_update shortcut");
17982        assert!(error.to_string().contains("unsupported_payload_codec"));
17983
17984        let unknown_query: QueryTask = serde_json::from_value(json!({
17985            "query_task_id": "query-unknown",
17986            "workflow_type": "missing",
17987            "query_name": "missing",
17988            "payload_codec": "json",
17989            "workflow_arguments": null,
17990            "query_arguments": null
17991        }))
17992        .expect("query task");
17993        let failure = worker
17994            .execute_query_task(unknown_query)
17995            .await
17996            .expect_err("codec must precede unknown query outcome");
17997        assert_eq!(failure.reason, "query_payload_decode_failed");
17998        assert!(failure.message.contains("unsupported_payload_codec"));
17999    }
18000
18001    #[tokio::test]
18002    async fn invalid_signal_history_payload_aliases_precede_shortcuts() {
18003        let client = Client::new("http://127.0.0.1:8080").expect("client");
18004        let worker = Worker::new(client, "rust-workers");
18005
18006        for event_type in ["SignalReceived", "SignalApplied"] {
18007            for (payload_field, codec) in [
18008                ("value", "json"),
18009                ("input", "unknown"),
18010                ("arguments", "json"),
18011            ] {
18012                let payload = json!({
18013                    "signal_name": "empty",
18014                    payload_field: {"codec": codec, "blob": null}
18015                });
18016                let workflow = workflow_task(
18017                    "missing",
18018                    vec![history_event(event_type, payload.clone())],
18019                    DEFAULT_CODEC,
18020                );
18021                let error = worker
18022                    .execute_workflow_task(workflow)
18023                    .expect_err("signal payload codec must precede unknown workflow outcome");
18024                assert!(
18025                    error.to_string().contains("unsupported_payload_codec"),
18026                    "{event_type}.{payload_field} returned an unrelated workflow error: {error}"
18027                );
18028
18029                let query: QueryTask = serde_json::from_value(json!({
18030                    "query_task_id": format!("query-{event_type}-{payload_field}"),
18031                    "workflow_type": "missing",
18032                    "query_name": "missing",
18033                    "payload_codec": DEFAULT_CODEC,
18034                    "workflow_arguments": null,
18035                    "query_arguments": null,
18036                    "history_events": [{
18037                        "event_type": event_type,
18038                        "payload": payload
18039                    }]
18040                }))
18041                .expect("query task");
18042                let failure = worker
18043                    .execute_query_task(query)
18044                    .await
18045                    .expect_err("signal payload codec must precede unknown query outcome");
18046                assert_eq!(
18047                    failure.reason, "query_payload_decode_failed",
18048                    "{event_type}.{payload_field} returned an unrelated query outcome"
18049                );
18050                assert!(
18051                    failure.message.contains("unsupported_payload_codec"),
18052                    "{event_type}.{payload_field} returned an unrelated query error: {}",
18053                    failure.message
18054                );
18055            }
18056        }
18057    }
18058
18059    #[test]
18060    fn workflow_context_schedules_activity_until_completion_is_in_history() {
18061        let ctx = WorkflowContext {
18062            state: Arc::new(Mutex::new(
18063                WorkflowState::new_with_identity(
18064                    Vec::new(),
18065                    Some("wf-parent".to_string()),
18066                    Some("run-parent".to_string()),
18067                    "rust-workers".to_string(),
18068                    DEFAULT_CODEC.to_string(),
18069                    None,
18070                )
18071                .expect("workflow state"),
18072            )),
18073        };
18074
18075        let mut call = Box::pin(ctx.activity("hello.activity", json!(["Ada"])));
18076        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18077        assert!(matches!(
18078            call.as_mut().poll(&mut task_context),
18079            Poll::Pending
18080        ));
18081
18082        let commands = ctx.take_commands().expect("commands");
18083        assert_eq!(commands[0]["type"], "schedule_activity");
18084        assert_eq!(commands[0]["activity_type"], "hello.activity");
18085    }
18086
18087    #[test]
18088    fn activity_options_encode_retry_policy_queue_and_every_timeout() {
18089        let ctx = workflow_context(Vec::new());
18090        let options = ActivityOptions::new()
18091            .task_queue("payments")
18092            .retry_policy(
18093                ActivityRetryPolicy::new(4)
18094                    .exponential_backoff(Duration::from_secs(1), 3, Some(Duration::from_secs(10)))
18095                    .non_retryable_error_type("ValidationError"),
18096            )
18097            .start_to_close_timeout(Duration::from_secs(120))
18098            .schedule_to_start_timeout(Duration::from_secs(10))
18099            .schedule_to_close_timeout(Duration::from_secs(300))
18100            .heartbeat_timeout(Duration::from_secs(15));
18101        let mut call = Box::pin(ctx.activity_with_options(
18102            "charge-card",
18103            options,
18104            json!([{"order_id": "o-1"}]),
18105        ));
18106        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18107
18108        assert!(matches!(
18109            call.as_mut().poll(&mut task_context),
18110            Poll::Pending
18111        ));
18112        assert!(matches!(
18113            call.as_mut().poll(&mut task_context),
18114            Poll::Pending
18115        ));
18116
18117        let commands = ctx.take_commands().expect("activity command");
18118        assert_eq!(commands.len(), 1, "one future emits one logical schedule");
18119        assert_eq!(commands[0]["queue"], "payments");
18120        assert_eq!(
18121            commands[0]["retry_policy"],
18122            json!({
18123                "max_attempts": 4,
18124                "backoff_seconds": [1, 3, 9],
18125                "non_retryable_error_types": ["ValidationError"],
18126            })
18127        );
18128        assert_eq!(commands[0]["start_to_close_timeout"], 120);
18129        assert_eq!(commands[0]["schedule_to_start_timeout"], 10);
18130        assert_eq!(commands[0]["schedule_to_close_timeout"], 300);
18131        assert_eq!(commands[0]["heartbeat_timeout"], 15);
18132    }
18133
18134    #[test]
18135    fn activity_options_encode_explicit_and_rounded_backoff_intervals() {
18136        let ctx = workflow_context(Vec::new());
18137        let options = ActivityOptions::new().retry_policy(
18138            ActivityRetryPolicy::new(3)
18139                .backoff_intervals([Duration::from_millis(1), Duration::from_millis(1_001)]),
18140        );
18141        let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18142        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18143
18144        assert!(matches!(
18145            call.as_mut().poll(&mut task_context),
18146            Poll::Pending
18147        ));
18148        assert_eq!(
18149            ctx.take_commands().expect("command")[0]["retry_policy"]["backoff_seconds"],
18150            json!([1, 2])
18151        );
18152    }
18153
18154    #[test]
18155    fn invalid_activity_options_return_typed_errors_before_emitting_commands() {
18156        let cases = [
18157            (
18158                ActivityOptions::new().task_queue("  "),
18159                ActivityOptionsErrorKind::EmptyTaskQueue,
18160            ),
18161            (
18162                ActivityOptions::new().retry_policy(ActivityRetryPolicy::default()),
18163                ActivityOptionsErrorKind::EmptyRetryPolicy,
18164            ),
18165            (
18166                ActivityOptions::new().retry_policy(ActivityRetryPolicy::new(0)),
18167                ActivityOptionsErrorKind::InvalidMaxAttempts,
18168            ),
18169            (
18170                ActivityOptions::new().retry_policy(ActivityRetryPolicy {
18171                    max_attempts: None,
18172                    backoff: Some(ActivityBackoff::Explicit(vec![Duration::from_secs(1)])),
18173                    non_retryable_error_types: Vec::new(),
18174                }),
18175                ActivityOptionsErrorKind::BackoffWithoutRetryBudget,
18176            ),
18177            (
18178                ActivityOptions::new().retry_policy(
18179                    ActivityRetryPolicy::new(2)
18180                        .backoff_intervals([Duration::from_secs(1), Duration::from_secs(2)]),
18181                ),
18182                ActivityOptionsErrorKind::TooManyBackoffIntervals,
18183            ),
18184            (
18185                ActivityOptions::new().retry_policy(
18186                    ActivityRetryPolicy::new(2).exponential_backoff(
18187                        Duration::from_secs(1),
18188                        0,
18189                        None,
18190                    ),
18191                ),
18192                ActivityOptionsErrorKind::InvalidBackoffCoefficient,
18193            ),
18194            (
18195                ActivityOptions::new()
18196                    .retry_policy(ActivityRetryPolicy::new(2).non_retryable_error_type("  ")),
18197                ActivityOptionsErrorKind::EmptyNonRetryableErrorType,
18198            ),
18199            (
18200                ActivityOptions::new().retry_policy(
18201                    ActivityRetryPolicy::new(10_002).exponential_backoff(
18202                        Duration::from_secs(1),
18203                        1,
18204                        None,
18205                    ),
18206                ),
18207                ActivityOptionsErrorKind::BackoffGenerationTooLarge,
18208            ),
18209            (
18210                ActivityOptions::new().retry_policy(
18211                    ActivityRetryPolicy::new(2)
18212                        .backoff_intervals([Duration::from_secs(i64::MAX as u64 + 1)]),
18213                ),
18214                ActivityOptionsErrorKind::BackoffOverflow,
18215            ),
18216        ];
18217
18218        for (options, expected_kind) in cases {
18219            let ctx = workflow_context(Vec::new());
18220            let mut call = Box::pin(ctx.activity_with_options("work", options, json!([])));
18221            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18222            let Poll::Ready(Err(Error::InvalidActivityOptions(error))) =
18223                call.as_mut().poll(&mut task_context)
18224            else {
18225                panic!("expected typed activity validation error");
18226            };
18227            assert_eq!(error.kind, expected_kind);
18228            assert!(ctx.take_commands().expect("commands").is_empty());
18229        }
18230    }
18231
18232    #[test]
18233    fn activity_options_validate_positive_and_ordered_timeouts() {
18234        let zero_timeout_cases = [
18235            ActivityOptions::new().start_to_close_timeout(Duration::ZERO),
18236            ActivityOptions::new().schedule_to_start_timeout(Duration::ZERO),
18237            ActivityOptions::new().schedule_to_close_timeout(Duration::ZERO),
18238            ActivityOptions::new().heartbeat_timeout(Duration::ZERO),
18239        ];
18240        for options in zero_timeout_cases {
18241            assert_eq!(
18242                options.validate().expect_err("zero timeout").kind,
18243                ActivityOptionsErrorKind::TimeoutNotPositive
18244            );
18245        }
18246
18247        let ordering_cases = [
18248            ActivityOptions::new()
18249                .heartbeat_timeout(Duration::from_secs(11))
18250                .start_to_close_timeout(Duration::from_secs(10)),
18251            ActivityOptions::new()
18252                .start_to_close_timeout(Duration::from_secs(31))
18253                .schedule_to_close_timeout(Duration::from_secs(30)),
18254            ActivityOptions::new()
18255                .schedule_to_start_timeout(Duration::from_secs(31))
18256                .schedule_to_close_timeout(Duration::from_secs(30)),
18257        ];
18258        for options in ordering_cases {
18259            assert_eq!(
18260                options.validate().expect_err("timeout order").kind,
18261                ActivityOptionsErrorKind::TimeoutOrder
18262            );
18263        }
18264
18265        assert_eq!(
18266            ActivityOptions::new()
18267                .start_to_close_timeout(Duration::from_secs(i64::MAX as u64 + 1))
18268                .validate()
18269                .expect_err("protocol integer overflow")
18270                .kind,
18271            ActivityOptionsErrorKind::TimeoutOverflow
18272        );
18273    }
18274
18275    #[test]
18276    fn replayed_activity_retry_history_completes_without_duplicate_schedule() {
18277        let ctx = workflow_context(completed_retry_activity_history());
18278        let mut call =
18279            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18280        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18281
18282        assert!(matches!(
18283            call.as_mut().poll(&mut task_context),
18284            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18285        ));
18286        assert!(ctx.take_commands().expect("commands").is_empty());
18287        ctx.ensure_history_consumed().expect("history consumed");
18288    }
18289
18290    #[test]
18291    fn duplicate_non_retryable_types_use_one_command_and_replay_representation() {
18292        let mut options = retry_activity_options();
18293        options
18294            .retry_policy
18295            .as_mut()
18296            .expect("retry policy")
18297            .non_retryable_error_types
18298            .extend([" PermanentError ".to_string(), "PermanentError".to_string()]);
18299
18300        let new_ctx = workflow_context(Vec::new());
18301        let mut new_call =
18302            Box::pin(new_ctx.activity_with_options("flaky", options.clone(), json!([])));
18303        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18304        assert!(matches!(
18305            new_call.as_mut().poll(&mut task_context),
18306            Poll::Pending
18307        ));
18308        let commands = new_ctx.take_commands().expect("commands");
18309        assert_eq!(commands.len(), 1);
18310        assert_eq!(
18311            commands[0]["retry_policy"]["non_retryable_error_types"],
18312            json!(["PermanentError"])
18313        );
18314
18315        let replay_ctx = workflow_context(completed_retry_activity_history());
18316        let mut replay_call =
18317            Box::pin(replay_ctx.activity_with_options("flaky", options, json!([])));
18318        assert!(matches!(
18319            replay_call.as_mut().poll(&mut task_context),
18320            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18321        ));
18322        assert!(replay_ctx.take_commands().expect("commands").is_empty());
18323        replay_ctx
18324            .ensure_history_consumed()
18325            .expect("history consumed");
18326    }
18327
18328    #[test]
18329    fn replayed_intermediate_retry_remains_pending_across_restarts() {
18330        let history = completed_retry_activity_history()
18331            .into_iter()
18332            .take(3)
18333            .collect::<Vec<_>>();
18334
18335        for _restart in 0..2 {
18336            let ctx = workflow_context(history.clone());
18337            let mut call =
18338                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18339            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18340            assert!(matches!(
18341                call.as_mut().poll(&mut task_context),
18342                Poll::Pending
18343            ));
18344            assert!(ctx.take_commands().expect("commands").is_empty());
18345        }
18346    }
18347
18348    #[test]
18349    fn replayed_activity_rejects_changed_queue_retry_and_every_timeout_field() {
18350        let mut changed_queue = retry_activity_options();
18351        changed_queue.task_queue = Some("different-queue".to_string());
18352
18353        let mut changed_max_attempts = retry_activity_options();
18354        let retry_policy = changed_max_attempts
18355            .retry_policy
18356            .as_mut()
18357            .expect("retry policy");
18358        retry_policy.max_attempts = Some(4);
18359
18360        let mut changed_backoff = retry_activity_options();
18361        let retry_policy = changed_backoff.retry_policy.as_mut().expect("retry policy");
18362        retry_policy.backoff = Some(ActivityBackoff::Explicit(vec![
18363            Duration::from_secs(3),
18364            Duration::from_secs(4),
18365        ]));
18366
18367        let mut changed_non_retryable_types = retry_activity_options();
18368        let retry_policy = changed_non_retryable_types
18369            .retry_policy
18370            .as_mut()
18371            .expect("retry policy");
18372        retry_policy.non_retryable_error_types = vec!["AnotherPermanentError".to_string()];
18373
18374        let mut changed_start_to_close = retry_activity_options();
18375        changed_start_to_close.start_to_close_timeout = Some(Duration::from_secs(31));
18376        let mut changed_schedule_to_start = retry_activity_options();
18377        changed_schedule_to_start.schedule_to_start_timeout = Some(Duration::from_secs(6));
18378        let mut changed_schedule_to_close = retry_activity_options();
18379        changed_schedule_to_close.schedule_to_close_timeout = Some(Duration::from_secs(91));
18380        let mut changed_heartbeat = retry_activity_options();
18381        changed_heartbeat.heartbeat_timeout = Some(Duration::from_secs(11));
18382
18383        let cases = [
18384            (changed_queue, "activity_task_queue_mismatch"),
18385            (changed_max_attempts, "activity_retry_policy_mismatch"),
18386            (changed_backoff, "activity_retry_policy_mismatch"),
18387            (
18388                changed_non_retryable_types,
18389                "activity_retry_policy_mismatch",
18390            ),
18391            (changed_start_to_close, "activity_retry_policy_mismatch"),
18392            (changed_schedule_to_start, "activity_retry_policy_mismatch"),
18393            (changed_schedule_to_close, "activity_retry_policy_mismatch"),
18394            (changed_heartbeat, "activity_retry_policy_mismatch"),
18395        ];
18396
18397        for (options, expected_reason) in cases {
18398            let ctx = workflow_context(completed_retry_activity_history());
18399            let mut call = Box::pin(ctx.activity_with_options("flaky", options, json!([])));
18400            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18401            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18402                call.as_mut().poll(&mut task_context)
18403            else {
18404                panic!("changed activity options must fail replay");
18405            };
18406            assert_eq!(failure.reason, expected_reason);
18407            assert_eq!(failure.sequence, Some(1));
18408            assert!(ctx.take_commands().expect("commands").is_empty());
18409        }
18410    }
18411
18412    #[test]
18413    fn replayed_activity_rejects_changed_execution_mode_and_snapshot_version() {
18414        let cases = [
18415            (
18416                "execution_mode",
18417                json!("local"),
18418                "activity_execution_mode_mismatch",
18419            ),
18420            (
18421                "snapshot_version",
18422                json!(2),
18423                "activity_retry_policy_mismatch",
18424            ),
18425        ];
18426
18427        for (field, value, expected_reason) in cases {
18428            let mut history = completed_retry_activity_history();
18429            let activity = history[0].payload["activity"]
18430                .as_object_mut()
18431                .expect("activity snapshot");
18432            if field == "execution_mode" {
18433                activity.insert(field.to_string(), value);
18434            } else {
18435                activity["retry_policy"]
18436                    .as_object_mut()
18437                    .expect("retry snapshot")
18438                    .insert(field.to_string(), value);
18439            }
18440
18441            let ctx = workflow_context(history);
18442            let mut call =
18443                Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18444            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18445            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18446                call.as_mut().poll(&mut task_context)
18447            else {
18448                panic!("changed {field} must fail replay");
18449            };
18450            assert_eq!(failure.reason, expected_reason);
18451            assert_eq!(failure.sequence, Some(1));
18452            assert!(ctx.take_commands().expect("commands").is_empty());
18453        }
18454    }
18455
18456    #[test]
18457    fn replayed_legacy_activity_treats_missing_option_snapshot_as_unknown() {
18458        let mut history = completed_retry_activity_history();
18459        let activity = history[0].payload["activity"]
18460            .as_object_mut()
18461            .expect("activity snapshot");
18462        activity.remove("execution_mode");
18463        activity.remove("retry_policy");
18464
18465        let mut current = retry_activity_options();
18466        current.start_to_close_timeout = Some(Duration::from_secs(45));
18467        current.schedule_to_start_timeout = Some(Duration::from_secs(8));
18468        current.schedule_to_close_timeout = Some(Duration::from_secs(120));
18469        current.heartbeat_timeout = Some(Duration::from_secs(12));
18470
18471        let ctx = workflow_context(history);
18472        let mut call = Box::pin(ctx.activity_with_options("flaky", current, json!([])));
18473        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18474        assert!(matches!(
18475            call.as_mut().poll(&mut task_context),
18476            Poll::Ready(Ok(result)) if result == json!({"status": "recovered"})
18477        ));
18478        assert!(ctx.take_commands().expect("commands").is_empty());
18479        ctx.ensure_history_consumed().expect("history consumed");
18480    }
18481
18482    #[test]
18483    fn terminal_activity_failed_after_start_returns_typed_failure() {
18484        let history = vec![
18485            history_event(
18486                "ActivityScheduled",
18487                json!({
18488                    "sequence": 1,
18489                    "activity_type": "flaky",
18490                    "activity_execution_id": "act-terminal",
18491                    "activity": {
18492                        "id": "act-terminal",
18493                        "sequence": 1,
18494                        "type": "flaky",
18495                        "queue": "critical-activities",
18496                        "retry_policy": {
18497                            "snapshot_version": 1,
18498                            "max_attempts": 3,
18499                            "backoff_seconds": [2, 4],
18500                            "non_retryable_error_types": ["PermanentError"]
18501                        }
18502                    }
18503                }),
18504            ),
18505            history_event(
18506                "ActivityStarted",
18507                json!({
18508                    "sequence": 1,
18509                    "activity_type": "flaky",
18510                    "activity_execution_id": "act-terminal",
18511                    "activity_attempt_id": "attempt-1",
18512                    "attempt_number": 1
18513                }),
18514            ),
18515            history_event(
18516                "ActivityFailed",
18517                json!({
18518                    "sequence": 1,
18519                    "activity_type": "flaky",
18520                    "activity_execution_id": "act-terminal",
18521                    "activity_attempt_id": "attempt-1",
18522                    "attempt_number": 1,
18523                    "failure_id": "failure-terminal",
18524                    "failure_category": "activity",
18525                    "exception_type": "PermanentError",
18526                    "message": "cannot retry",
18527                    "non_retryable": true
18528                }),
18529            ),
18530        ];
18531        let ctx = workflow_context(history);
18532        let mut call =
18533            Box::pin(ctx.activity_with_options("flaky", retry_activity_options(), json!([])));
18534        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18535
18536        let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18537            call.as_mut().poll(&mut task_context)
18538        else {
18539            panic!("terminal ActivityFailed must settle the activity future");
18540        };
18541        assert_eq!(failure.kind, ActivityFailureKind::Failed);
18542        assert_eq!(
18543            failure.activity_execution_id.as_deref(),
18544            Some("act-terminal")
18545        );
18546        assert_eq!(failure.exception_type.as_deref(), Some("PermanentError"));
18547        assert!(failure.non_retryable);
18548        assert!(ctx.take_commands().expect("commands").is_empty());
18549        ctx.ensure_history_consumed().expect("history consumed");
18550    }
18551
18552    #[test]
18553    fn activity_terminal_events_return_machine_readable_failures() {
18554        let cases = [
18555            (
18556                "ActivityFailed",
18557                json!({
18558                    "sequence": 1,
18559                    "activity_type": "charge-card",
18560                    "activity_execution_id": "act-1",
18561                    "activity_attempt_id": "attempt-2",
18562                    "attempt_number": 2,
18563                    "failure_id": "failure-1",
18564                    "failure_category": "activity",
18565                    "exception_type": "PaymentDeclined",
18566                    "exception_class": "payments.PaymentDeclined",
18567                    "message": "card declined",
18568                    "non_retryable": true
18569                }),
18570                ActivityFailureKind::Failed,
18571                "activity",
18572            ),
18573            (
18574                "ActivityCancelled",
18575                json!({
18576                    "sequence": 1,
18577                    "activity_type": "charge-card",
18578                    "activity_execution_id": "act-1",
18579                    "activity_attempt_id": "attempt-1"
18580                }),
18581                ActivityFailureKind::Cancelled,
18582                "cancelled",
18583            ),
18584        ];
18585
18586        for (event_type, payload, expected_kind, expected_reason) in cases {
18587            let ctx = workflow_context(vec![history_event(event_type, payload)]);
18588            let mut call = Box::pin(ctx.activity("charge-card", json!([])));
18589            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18590            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18591                call.as_mut().poll(&mut task_context)
18592            else {
18593                panic!("expected terminal activity failure");
18594            };
18595            assert_eq!(failure.kind, expected_kind);
18596            assert_eq!(failure.reason, expected_reason);
18597            assert_eq!(failure.activity_execution_id.as_deref(), Some("act-1"));
18598            assert_eq!(failure.activity_type.as_deref(), Some("charge-card"));
18599        }
18600    }
18601
18602    #[test]
18603    fn every_activity_timeout_class_is_typed() {
18604        for timeout_kind in [
18605            "start_to_close",
18606            "schedule_to_start",
18607            "schedule_to_close",
18608            "heartbeat",
18609        ] {
18610            let ctx = workflow_context(vec![history_event(
18611                "ActivityTimedOut",
18612                json!({
18613                    "sequence": 1,
18614                    "activity_type": "slow",
18615                    "activity_execution_id": "act-timeout",
18616                    "activity_attempt_id": "attempt-timeout",
18617                    "failure_category": "timeout",
18618                    "timeout_kind": timeout_kind,
18619                    "message": "deadline expired"
18620                }),
18621            )]);
18622            let mut call = Box::pin(ctx.activity("slow", json!([])));
18623            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18624            let Poll::Ready(Err(Error::ActivityFailed(failure))) =
18625                call.as_mut().poll(&mut task_context)
18626            else {
18627                panic!("expected timeout failure");
18628            };
18629            assert_eq!(failure.kind, ActivityFailureKind::TimedOut);
18630            assert_eq!(failure.reason, timeout_kind);
18631            assert_eq!(failure.timeout_kind.as_deref(), Some(timeout_kind));
18632            assert_eq!(failure.failure_category.as_deref(), Some("timeout"));
18633        }
18634    }
18635
18636    #[test]
18637    fn workflow_sleep_emits_one_durable_timer_and_rounds_up() {
18638        let ctx = workflow_context(Vec::new());
18639        let mut sleep = Box::pin(ctx.sleep(Duration::from_millis(1_001)));
18640        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18641
18642        assert!(matches!(
18643            sleep.as_mut().poll(&mut task_context),
18644            Poll::Pending
18645        ));
18646        assert!(matches!(
18647            sleep.as_mut().poll(&mut task_context),
18648            Poll::Pending
18649        ));
18650
18651        let commands = ctx.take_commands().expect("timer command");
18652        assert_eq!(
18653            commands,
18654            vec![json!({
18655                "type": "start_timer",
18656                "delay_seconds": 2,
18657            })]
18658        );
18659    }
18660
18661    #[test]
18662    fn workflow_sleep_replays_matching_schedule_and_fire_without_a_command() {
18663        let history = vec![
18664            history_event(
18665                "TimerScheduled",
18666                json!({
18667                    "sequence": 1,
18668                    "timer_id": "timer-1",
18669                    "delay_seconds": 5,
18670                    "fire_at": "2026-07-11T12:00:05Z",
18671                }),
18672            ),
18673            history_event(
18674                "TimerFired",
18675                json!({
18676                    "sequence": 1,
18677                    "timer_id": "timer-1",
18678                    "delay_seconds": 5,
18679                    "fire_at": "2026-07-11T12:00:05Z",
18680                    "fired_at": "2026-07-11T12:00:05Z",
18681                }),
18682            ),
18683        ];
18684
18685        for _restart in 0..2 {
18686            let ctx = workflow_context(history.clone());
18687            let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(5)));
18688            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18689            assert!(matches!(
18690                sleep.as_mut().poll(&mut task_context),
18691                Poll::Ready(Ok(()))
18692            ));
18693            assert!(ctx.take_commands().expect("commands").is_empty());
18694            ctx.ensure_history_consumed().expect("history consumed");
18695        }
18696    }
18697
18698    #[test]
18699    fn workflow_sleep_rejects_changed_delay_during_replay() {
18700        let ctx = workflow_context(vec![
18701            history_event(
18702                "TimerScheduled",
18703                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18704            ),
18705            history_event(
18706                "TimerFired",
18707                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
18708            ),
18709        ]);
18710        let mut sleep = Box::pin(ctx.sleep(Duration::from_secs(500)));
18711        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18712
18713        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
18714            sleep.as_mut().poll(&mut task_context)
18715        else {
18716            panic!("changed timer delay must be rejected");
18717        };
18718        assert_eq!(failure.reason, "timer_delay_mismatch");
18719        assert_eq!(failure.sequence, Some(1));
18720    }
18721
18722    #[test]
18723    fn workflow_condition_wait_emits_published_identity_and_timeout_contract() {
18724        let ctx = workflow_context(Vec::new());
18725        let mut wait = Box::pin(
18726            ctx.wait_condition(
18727                ConditionWaitOptions::new("approval.ready", "sha256:approval-v1")
18728                    .timeout(Duration::from_millis(60_001)),
18729                || Ok(false),
18730            ),
18731        );
18732        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18733
18734        assert!(matches!(
18735            wait.as_mut().poll(&mut task_context),
18736            Poll::Pending
18737        ));
18738        assert!(matches!(
18739            wait.as_mut().poll(&mut task_context),
18740            Poll::Pending
18741        ));
18742        assert_eq!(
18743            ctx.take_commands().expect("condition command"),
18744            vec![json!({
18745                "type": "open_condition_wait",
18746                "condition_wait_occurrence_id": "rust:condition-wait:0",
18747                "condition_key": "approval.ready",
18748                "condition_definition_fingerprint": "sha256:approval-v1",
18749                "timeout_seconds": 61,
18750            })]
18751        );
18752    }
18753
18754    #[test]
18755    fn workflow_condition_wait_returns_explicit_immediate_results_without_commands() {
18756        let ctx = workflow_context(Vec::new());
18757        let mut satisfied = Box::pin(wait_condition!(ctx, "already-ready", || Ok(true)));
18758        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18759        assert!(matches!(
18760            satisfied.as_mut().poll(&mut task_context),
18761            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18762        ));
18763
18764        let mut timed_out = Box::pin(wait_condition!(
18765            ctx,
18766            "no-wait",
18767            timeout: Duration::ZERO,
18768            || Ok(false),
18769        ));
18770        assert!(matches!(
18771            timed_out.as_mut().poll(&mut task_context),
18772            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18773        ));
18774        assert!(ctx.take_commands().expect("commands").is_empty());
18775    }
18776
18777    #[test]
18778    fn signal_and_update_history_reevaluate_open_conditions_after_restart() {
18779        let signal_history = vec![
18780            history_event(
18781                "ConditionWaitOpened",
18782                json!({
18783                    "sequence": 4,
18784                    "condition_wait_id": "condition:4",
18785                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18786                    "condition_key": "approval",
18787                    "condition_definition_fingerprint": "sha256:approval-v1",
18788                    "timeout_seconds": 30,
18789                }),
18790            ),
18791            history_event(
18792                "SignalReceived",
18793                json!({
18794                    "workflow_sequence": 4,
18795                    "signal_name": "approve",
18796                    "arguments": fixture_envelope(json!(["Ada"])),
18797                }),
18798            ),
18799        ];
18800        for _worker_before_or_after_restart in 0..2 {
18801            let ctx = workflow_context(signal_history.clone());
18802            let predicate_ctx = ctx.clone();
18803            let mut wait = Box::pin(
18804                ctx.wait_condition(
18805                    ConditionWaitOptions::new("approval", "sha256:approval-v1")
18806                        .timeout(Duration::from_secs(30)),
18807                    move || Ok(!predicate_ctx.signals("approve")?.is_empty()),
18808                ),
18809            );
18810            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18811            assert!(matches!(
18812                wait.as_mut().poll(&mut task_context),
18813                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18814            ));
18815            assert!(ctx.take_commands().expect("commands").is_empty());
18816            ctx.ensure_history_consumed().expect("condition consumed");
18817        }
18818
18819        let update_history = vec![
18820            history_event(
18821                "ConditionWaitOpened",
18822                json!({
18823                    "sequence": 7,
18824                    "condition_wait_id": "condition:7",
18825                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18826                    "condition_key": "update-approval",
18827                    "condition_definition_fingerprint": "sha256:update-approval-v1",
18828                }),
18829            ),
18830            history_event(
18831                "UpdateApplied",
18832                json!({
18833                    "sequence": 7,
18834                    "update_id": "update-1",
18835                    "update_name": "approve",
18836                    "arguments": fixture_envelope(json!([true])),
18837                }),
18838            ),
18839        ];
18840        let ctx = workflow_context(update_history);
18841        let predicate_ctx = ctx.clone();
18842        let mut wait = Box::pin(ctx.wait_condition(
18843            ConditionWaitOptions::new("update-approval", "sha256:update-approval-v1"),
18844            move || {
18845                Ok(predicate_ctx
18846                    .updates("approve")?
18847                    .first()
18848                    .and_then(|arguments| arguments.first())
18849                    .and_then(Value::as_bool)
18850                    == Some(true))
18851            },
18852        ));
18853        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18854        assert!(matches!(
18855            wait.as_mut().poll(&mut task_context),
18856            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18857        ));
18858        assert!(ctx.take_commands().expect("commands").is_empty());
18859        ctx.ensure_history_consumed().expect("condition consumed");
18860    }
18861
18862    #[test]
18863    fn condition_wait_preserves_open_satisfied_and_timed_out_replay_states() {
18864        let open_history = vec![
18865            history_event(
18866                "ConditionWaitOpened",
18867                json!({
18868                    "sequence": 3,
18869                    "condition_wait_id": "condition:3",
18870                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18871                    "condition_key": "two-votes",
18872                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18873                    "timeout_seconds": 120,
18874                }),
18875            ),
18876            history_event(
18877                "SignalReceived",
18878                json!({
18879                    "workflow_sequence": 3,
18880                    "signal_name": "vote",
18881                    "arguments": fixture_envelope(json!(["first"])),
18882                }),
18883            ),
18884        ];
18885        for _worker_before_or_after_restart in 0..2 {
18886            let ctx = workflow_context(open_history.clone());
18887            let predicate_ctx = ctx.clone();
18888            let mut wait = Box::pin(
18889                ctx.wait_condition(
18890                    ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1")
18891                        .timeout(Duration::from_secs(120)),
18892                    move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
18893                ),
18894            );
18895            let mut task_context = TaskContext::from_waker(noop_waker_ref());
18896            assert!(matches!(
18897                wait.as_mut().poll(&mut task_context),
18898                Poll::Pending
18899            ));
18900            assert_eq!(
18901                ctx.take_commands().expect("reopened condition"),
18902                vec![json!({
18903                    "type": "open_condition_wait",
18904                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18905                    "condition_key": "two-votes",
18906                    "condition_definition_fingerprint": "sha256:two-votes-v1",
18907                    "timeout_seconds": 120,
18908                })]
18909            );
18910        }
18911
18912        let satisfied_ctx = workflow_context(vec![
18913            history_event(
18914                "ConditionWaitOpened",
18915                json!({
18916                    "sequence": 5,
18917                    "condition_wait_id": "condition:5",
18918                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18919                    "condition_key": "approval",
18920                    "condition_definition_fingerprint": "sha256:approval-v1",
18921                }),
18922            ),
18923            history_event(
18924                "ConditionWaitSatisfied",
18925                json!({
18926                    "sequence": 5,
18927                    "condition_wait_id": "condition:5",
18928                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18929                    "condition_key": "approval",
18930                    "condition_definition_fingerprint": "sha256:approval-v1",
18931                }),
18932            ),
18933        ]);
18934        let mut satisfied = Box::pin(satisfied_ctx.wait_condition(
18935            ConditionWaitOptions::new("approval", "sha256:approval-v1"),
18936            || Ok(false),
18937        ));
18938        let mut task_context = TaskContext::from_waker(noop_waker_ref());
18939        assert!(matches!(
18940            satisfied.as_mut().poll(&mut task_context),
18941            Poll::Ready(Ok(ConditionWaitResult::Satisfied))
18942        ));
18943
18944        let timed_out_ctx = workflow_context(vec![
18945            history_event(
18946                "ConditionWaitOpened",
18947                json!({
18948                    "sequence": 8,
18949                    "condition_wait_id": "condition:8",
18950                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18951                    "condition_key": "approval-timeout",
18952                    "condition_definition_fingerprint": "sha256:approval-timeout-v1",
18953                    "timeout_seconds": 5,
18954                }),
18955            ),
18956            history_event(
18957                "TimerScheduled",
18958                json!({
18959                    "sequence": 9,
18960                    "timer_id": "condition-timer:9",
18961                    "timer_kind": "condition_timeout",
18962                    "condition_wait_id": "condition:8",
18963                    "delay_seconds": 5,
18964                }),
18965            ),
18966            history_event(
18967                "TimerFired",
18968                json!({
18969                    "sequence": 9,
18970                    "timer_id": "condition-timer:9",
18971                    "timer_kind": "condition_timeout",
18972                    "condition_wait_id": "condition:8",
18973                    "delay_seconds": 5,
18974                }),
18975            ),
18976        ]);
18977        let mut timed_out = Box::pin(
18978            timed_out_ctx.wait_condition(
18979                ConditionWaitOptions::new("approval-timeout", "sha256:approval-timeout-v1")
18980                    .timeout(Duration::from_secs(5)),
18981                || Ok(true),
18982            ),
18983        );
18984        assert!(matches!(
18985            timed_out.as_mut().poll(&mut task_context),
18986            Poll::Ready(Ok(ConditionWaitResult::TimedOut))
18987        ));
18988    }
18989
18990    #[test]
18991    fn condition_wait_replays_repeated_physical_opens_as_one_logical_wait() {
18992        let history = vec![
18993            history_event(
18994                "ConditionWaitOpened",
18995                json!({
18996                    "sequence": 3,
18997                    "condition_wait_id": "condition:3",
18998                    "condition_wait_occurrence_id": "rust:condition-wait:0",
18999                    "condition_key": "two-votes",
19000                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19001                }),
19002            ),
19003            history_event(
19004                "SignalReceived",
19005                json!({
19006                    "workflow_sequence": 3,
19007                    "signal_name": "vote",
19008                    "arguments": fixture_envelope(json!(["first"])),
19009                }),
19010            ),
19011            history_event(
19012                "ConditionWaitSatisfied",
19013                json!({
19014                    "sequence": 3,
19015                    "condition_wait_id": "condition:3",
19016                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19017                    "condition_key": "two-votes",
19018                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19019                }),
19020            ),
19021            history_event(
19022                "ConditionWaitOpened",
19023                json!({
19024                    "sequence": 5,
19025                    "condition_wait_id": "condition:5",
19026                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19027                    "condition_key": "two-votes",
19028                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19029                }),
19030            ),
19031            history_event(
19032                "SignalReceived",
19033                json!({
19034                    "workflow_sequence": 5,
19035                    "signal_name": "vote",
19036                    "arguments": fixture_envelope(json!(["second"])),
19037                }),
19038            ),
19039            history_event(
19040                "ConditionWaitSatisfied",
19041                json!({
19042                    "sequence": 5,
19043                    "condition_wait_id": "condition:5",
19044                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19045                    "condition_key": "two-votes",
19046                    "condition_definition_fingerprint": "sha256:two-votes-v1",
19047                }),
19048            ),
19049        ];
19050        for _cold_worker_or_restart in 0..2 {
19051            let ctx = workflow_context(history.clone());
19052            let predicate_ctx = ctx.clone();
19053            let mut wait = Box::pin(ctx.wait_condition(
19054                ConditionWaitOptions::new("two-votes", "sha256:two-votes-v1"),
19055                move || Ok(predicate_ctx.signals("vote")?.len() >= 2),
19056            ));
19057            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19058
19059            assert!(matches!(
19060                wait.as_mut().poll(&mut task_context),
19061                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19062            ));
19063            assert!(ctx.take_commands().expect("commands").is_empty());
19064            ctx.ensure_history_consumed()
19065                .expect("every physical wait-open is consumed");
19066        }
19067    }
19068
19069    #[test]
19070    fn condition_wait_replays_update_driven_physical_opens_as_one_occurrence() {
19071        let history = vec![
19072            history_event(
19073                "ConditionWaitOpened",
19074                json!({
19075                    "sequence": 3,
19076                    "condition_wait_id": "condition:3",
19077                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19078                    "condition_key": "approved",
19079                    "condition_definition_fingerprint": "sha256:approved-v1",
19080                }),
19081            ),
19082            history_event(
19083                "UpdateApplied",
19084                json!({
19085                    "sequence": 3,
19086                    "update_id": "update-1",
19087                    "update_name": "approve",
19088                    "arguments": fixture_envelope(json!([false])),
19089                }),
19090            ),
19091            history_event(
19092                "ConditionWaitOpened",
19093                json!({
19094                    "sequence": 5,
19095                    "condition_wait_id": "condition:5",
19096                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19097                    "condition_key": "approved",
19098                    "condition_definition_fingerprint": "sha256:approved-v1",
19099                }),
19100            ),
19101            history_event(
19102                "UpdateApplied",
19103                json!({
19104                    "sequence": 5,
19105                    "update_id": "update-2",
19106                    "update_name": "approve",
19107                    "arguments": fixture_envelope(json!([true])),
19108                }),
19109            ),
19110        ];
19111
19112        for _cold_worker_or_restart in 0..2 {
19113            let ctx = workflow_context(history.clone());
19114            let predicate_ctx = ctx.clone();
19115            let mut wait = Box::pin(ctx.wait_condition(
19116                ConditionWaitOptions::new("approved", "sha256:approved-v1"),
19117                move || {
19118                    Ok(predicate_ctx
19119                        .updates("approve")?
19120                        .last()
19121                        .and_then(|arguments| arguments.first())
19122                        .and_then(Value::as_bool)
19123                        == Some(true))
19124                },
19125            ));
19126            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19127
19128            assert!(matches!(
19129                wait.as_mut().poll(&mut task_context),
19130                Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19131            ));
19132            assert!(ctx.take_commands().expect("commands").is_empty());
19133            ctx.ensure_history_consumed()
19134                .expect("every update-driven reopen is consumed");
19135        }
19136    }
19137
19138    #[test]
19139    fn condition_wait_replay_keeps_every_adjacent_authored_occurrence_distinct() {
19140        for (first_key, first_fingerprint, second_key, second_fingerprint) in [
19141            ("shared", "sha256:first", "shared", "sha256:second"),
19142            ("first", "sha256:shared", "second", "sha256:shared"),
19143            ("shared", "sha256:shared", "shared", "sha256:shared"),
19144            ("first", "sha256:first", "second", "sha256:second"),
19145        ] {
19146            let history = vec![
19147                history_event(
19148                    "ConditionWaitOpened",
19149                    json!({
19150                        "sequence": 3,
19151                        "condition_wait_id": "condition:3",
19152                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19153                        "condition_key": first_key,
19154                        "condition_definition_fingerprint": first_fingerprint,
19155                    }),
19156                ),
19157                history_event(
19158                    "ConditionWaitSatisfied",
19159                    json!({
19160                        "sequence": 3,
19161                        "condition_wait_id": "condition:3",
19162                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19163                        "condition_key": first_key,
19164                        "condition_definition_fingerprint": first_fingerprint,
19165                    }),
19166                ),
19167                history_event(
19168                    "ConditionWaitOpened",
19169                    json!({
19170                        "sequence": 4,
19171                        "condition_wait_id": "condition:4",
19172                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19173                        "condition_key": second_key,
19174                        "condition_definition_fingerprint": second_fingerprint,
19175                    }),
19176                ),
19177                history_event(
19178                    "ConditionWaitSatisfied",
19179                    json!({
19180                        "sequence": 4,
19181                        "condition_wait_id": "condition:4",
19182                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19183                        "condition_key": second_key,
19184                        "condition_definition_fingerprint": second_fingerprint,
19185                    }),
19186                ),
19187            ];
19188            for _cold_worker_or_restart in 0..2 {
19189                let ctx = workflow_context(history.clone());
19190                let mut task_context = TaskContext::from_waker(noop_waker_ref());
19191                let mut first = Box::pin(ctx.wait_condition(
19192                    ConditionWaitOptions::new(first_key, first_fingerprint),
19193                    || Ok(false),
19194                ));
19195                assert!(matches!(
19196                    first.as_mut().poll(&mut task_context),
19197                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19198                ));
19199
19200                let mut second = Box::pin(ctx.wait_condition(
19201                    ConditionWaitOptions::new(second_key, second_fingerprint),
19202                    || Ok(false),
19203                ));
19204                assert!(matches!(
19205                    second.as_mut().poll(&mut task_context),
19206                    Poll::Ready(Ok(ConditionWaitResult::Satisfied))
19207                ));
19208                assert!(ctx.take_commands().expect("commands").is_empty());
19209                ctx.ensure_history_consumed()
19210                    .expect("each authored wait consumes one occurrence");
19211            }
19212        }
19213    }
19214
19215    #[test]
19216    fn cold_workers_replay_adjacent_condition_waits_from_one_loop_call_site() {
19217        fn worker() -> Worker {
19218            let client = Client::new("http://127.0.0.1:8080").expect("client");
19219            let mut worker = Worker::new(client, "rust-workers");
19220            worker.register_workflow("rust.condition-loop", |ctx, _input| async move {
19221                let mut outcomes = Vec::new();
19222                for _ in 0..2 {
19223                    outcomes.push(
19224                        ctx.wait_condition(
19225                            ConditionWaitOptions::new("shared", "sha256:shared"),
19226                            || Ok(false),
19227                        )
19228                        .await?,
19229                    );
19230                }
19231                Ok(json!(outcomes))
19232            });
19233            worker
19234        }
19235
19236        let task = workflow_task(
19237            "rust.condition-loop",
19238            vec![
19239                history_event(
19240                    "ConditionWaitOpened",
19241                    json!({
19242                        "sequence": 1,
19243                        "condition_wait_id": "condition:1",
19244                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19245                        "condition_key": "shared",
19246                        "condition_definition_fingerprint": "sha256:shared",
19247                    }),
19248                ),
19249                history_event(
19250                    "ConditionWaitSatisfied",
19251                    json!({
19252                        "sequence": 1,
19253                        "condition_wait_id": "condition:1",
19254                        "condition_wait_occurrence_id": "rust:condition-wait:0",
19255                        "condition_key": "shared",
19256                        "condition_definition_fingerprint": "sha256:shared",
19257                    }),
19258                ),
19259                history_event(
19260                    "ConditionWaitOpened",
19261                    json!({
19262                        "sequence": 2,
19263                        "condition_wait_id": "condition:2",
19264                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19265                        "condition_key": "shared",
19266                        "condition_definition_fingerprint": "sha256:shared",
19267                    }),
19268                ),
19269                history_event(
19270                    "ConditionWaitSatisfied",
19271                    json!({
19272                        "sequence": 2,
19273                        "condition_wait_id": "condition:2",
19274                        "condition_wait_occurrence_id": "rust:condition-wait:1",
19275                        "condition_key": "shared",
19276                        "condition_definition_fingerprint": "sha256:shared",
19277                    }),
19278                ),
19279            ],
19280            DEFAULT_CODEC,
19281        );
19282
19283        for _cold_worker_or_restart in 0..2 {
19284            let commands = worker()
19285                .execute_workflow_task(task.clone())
19286                .expect("adjacent loop waits replay deterministically");
19287            assert_eq!(commands.len(), 1);
19288            assert_eq!(commands[0]["type"], "complete_workflow");
19289            assert_eq!(
19290                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
19291                json!(["satisfied", "satisfied"])
19292            );
19293        }
19294    }
19295
19296    #[test]
19297    fn condition_wait_replay_rejects_identity_predicate_and_timeout_changes() {
19298        let history = vec![history_event(
19299            "ConditionWaitOpened",
19300            json!({
19301                "sequence": 12,
19302                "condition_wait_id": "condition:12",
19303                "condition_wait_occurrence_id": "rust:condition-wait:0",
19304                "condition_key": "approval",
19305                "condition_definition_fingerprint": "sha256:approval-v1",
19306                "timeout_seconds": 30,
19307            }),
19308        )];
19309        for (options, expected_reason) in [
19310            (
19311                ConditionWaitOptions::new("changed", "sha256:approval-v1")
19312                    .timeout(Duration::from_secs(30)),
19313                "condition_wait_key_mismatch",
19314            ),
19315            (
19316                ConditionWaitOptions::new("approval", "sha256:approval-v2")
19317                    .timeout(Duration::from_secs(30)),
19318                "condition_wait_predicate_mismatch",
19319            ),
19320            (
19321                ConditionWaitOptions::new("approval", "sha256:approval-v1")
19322                    .timeout(Duration::from_secs(29)),
19323                "condition_wait_timeout_mismatch",
19324            ),
19325        ] {
19326            let ctx = workflow_context(history.clone());
19327            let mut wait = Box::pin(ctx.wait_condition(options, || Ok(false)));
19328            let mut task_context = TaskContext::from_waker(noop_waker_ref());
19329            let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19330                wait.as_mut().poll(&mut task_context)
19331            else {
19332                panic!("changed condition definition must fail replay");
19333            };
19334            assert_eq!(failure.reason, expected_reason);
19335            assert_eq!(failure.sequence, Some(12));
19336        }
19337    }
19338
19339    #[test]
19340    fn condition_wait_history_requires_the_canonical_predicate_fingerprint() {
19341        let error = WorkflowState::new(
19342            vec![history_event(
19343                "ConditionWaitOpened",
19344                json!({
19345                    "sequence": 12,
19346                    "condition_wait_id": "condition:12",
19347                    "condition_wait_occurrence_id": "rust:condition-wait:0",
19348                    "condition_key": "approval",
19349                }),
19350            )],
19351            "rust-workers".to_string(),
19352            DEFAULT_CODEC.to_string(),
19353            None,
19354        )
19355        .expect_err("condition history without a predicate fingerprint must fail");
19356
19357        assert!(matches!(
19358            error,
19359            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19360                if reason == "condition_wait_predicate_fingerprint_missing"
19361        ));
19362    }
19363
19364    #[test]
19365    fn condition_wait_history_requires_authored_occurrence_identity() {
19366        let error = WorkflowState::new(
19367            vec![history_event(
19368                "ConditionWaitOpened",
19369                json!({
19370                    "sequence": 12,
19371                    "condition_wait_id": "condition:12",
19372                    "condition_key": "approval",
19373                    "condition_definition_fingerprint": "sha256:approval-v1",
19374                }),
19375            )],
19376            "rust-workers".to_string(),
19377            DEFAULT_CODEC.to_string(),
19378            None,
19379        )
19380        .expect_err("condition history without occurrence identity must fail");
19381
19382        assert!(matches!(
19383            error,
19384            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19385                if reason == "condition_wait_occurrence_id_missing"
19386        ));
19387    }
19388
19389    #[test]
19390    fn typed_search_attribute_updates_validate_emit_and_replay() {
19391        let update = SearchAttributeUpdate::new()
19392            .keyword("OrderStatus", " waiting ")
19393            .expect("keyword")
19394            .int("Attempt", 3)
19395            .expect("int")
19396            .bool("Escalated", false)
19397            .expect("bool")
19398            .keyword_list("Regions", ["us-east", "eu-west"])
19399            .expect("list")
19400            .datetime("UpdatedAt", "2026-08-22T04:00:00Z")
19401            .expect("datetime")
19402            .delete("LegacyStatus")
19403            .expect("delete");
19404        let ctx = workflow_context(Vec::new());
19405        ctx.upsert_search_attributes(update.clone())
19406            .expect("typed update");
19407        assert_eq!(
19408            ctx.take_commands().expect("search-attribute command"),
19409            vec![json!({
19410                "type": "upsert_search_attributes",
19411                "attributes": {
19412                    "Attempt": 3,
19413                    "Escalated": false,
19414                    "LegacyStatus": null,
19415                    "OrderStatus": "waiting",
19416                    "Regions": ["us-east", "eu-west"],
19417                    "UpdatedAt": "2026-08-22T04:00:00Z",
19418                },
19419                "attribute_types": {
19420                    "Attempt": "int",
19421                    "Escalated": "bool",
19422                    "OrderStatus": "keyword",
19423                    "Regions": "keyword_list",
19424                    "UpdatedAt": "datetime",
19425                },
19426            })]
19427        );
19428
19429        let replay = workflow_context(vec![history_event(
19430            "SearchAttributesUpserted",
19431            json!({
19432                "sequence": 6,
19433                "attributes": {
19434                    "Attempt": 3,
19435                    "Escalated": false,
19436                    "LegacyStatus": null,
19437                    "OrderStatus": "waiting",
19438                    "Regions": ["us-east", "eu-west"],
19439                    "UpdatedAt": "2026-08-22T04:00:00Z",
19440                },
19441                "attribute_types": {
19442                    "Attempt": "int",
19443                    "Escalated": "bool",
19444                    "OrderStatus": "keyword",
19445                    "Regions": "keyword_list",
19446                    "UpdatedAt": "datetime",
19447                },
19448                "merged": {},
19449            }),
19450        )]);
19451        replay
19452            .upsert_search_attributes(update)
19453            .expect("matching update replays");
19454        assert!(replay.take_commands().expect("commands").is_empty());
19455        replay.ensure_history_consumed().expect("history consumed");
19456
19457        let type_drift = workflow_context(vec![history_event(
19458            "SearchAttributesUpserted",
19459            json!({
19460                "sequence": 7,
19461                "attributes": {"OrderStatus": "waiting"},
19462                "attribute_types": {"OrderStatus": "keyword"},
19463                "merged": {"OrderStatus": "waiting"},
19464            }),
19465        )]);
19466        let error = type_drift
19467            .upsert_search_attributes(
19468                SearchAttributeUpdate::new()
19469                    .string("OrderStatus", "waiting")
19470                    .expect("string update"),
19471            )
19472            .expect_err("same JSON value with a changed type must fail replay");
19473        assert!(matches!(
19474            error,
19475            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19476                if reason == "search_attribute_type_mismatch"
19477        ));
19478
19479        let malformed_types = WorkflowState::new(
19480            vec![history_event(
19481                "SearchAttributesUpserted",
19482                json!({
19483                    "sequence": 8,
19484                    "attributes": {"OrderStatus": "waiting"},
19485                    "attribute_types": {"OrderStatus": "unsupported"},
19486                    "merged": {"OrderStatus": "waiting"},
19487                }),
19488            )],
19489            "rust-workers".to_string(),
19490            DEFAULT_CODEC.to_string(),
19491            None,
19492        )
19493        .expect_err("unsupported search-attribute type metadata must fail");
19494        assert!(matches!(
19495            malformed_types,
19496            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19497                if reason == "search_attribute_types_malformed"
19498        ));
19499
19500        assert!(matches!(
19501            SearchAttributeUpdate::new().keyword("bad key", "value"),
19502            Err(SearchAttributeUpdateError::InvalidKey(_))
19503        ));
19504        assert!(matches!(
19505            SearchAttributeUpdate::new().float("Ratio", f64::NAN),
19506            Err(SearchAttributeUpdateError::NonFiniteFloat(_))
19507        ));
19508        assert!(matches!(
19509            SearchAttributeUpdate::new().keyword("UnicodeKeyword", "é".repeat(128)),
19510            Err(SearchAttributeUpdateError::ValueTooLong { .. })
19511        ));
19512        assert!(matches!(
19513            SearchAttributeUpdate::new().datetime("UpdatedAt", "2026-02-30T04:00:00Z"),
19514            Err(SearchAttributeUpdateError::InvalidDateTime(_))
19515        ));
19516        assert!(matches!(
19517            workflow_context(Vec::new()).upsert_search_attributes(SearchAttributeUpdate::new()),
19518            Err(Error::InvalidSearchAttributeUpdate(
19519                SearchAttributeUpdateError::Empty
19520            ))
19521        ));
19522    }
19523
19524    #[test]
19525    fn typed_search_attribute_text_uses_the_runtime_byte_limit() {
19526        let ascii = "a".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH);
19527        let utf8 = "é".repeat(MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2);
19528
19529        assert!(SearchAttributeUpdate::new()
19530            .string("AsciiDescription", ascii)
19531            .is_ok());
19532        assert!(SearchAttributeUpdate::new()
19533            .string("Utf8Description", utf8)
19534            .is_ok());
19535        assert!(matches!(
19536            SearchAttributeUpdate::new().string(
19537                "TooLongDescription",
19538                "é".repeat((MAX_SEARCH_ATTRIBUTE_STRING_LENGTH / 2) + 1),
19539            ),
19540            Err(SearchAttributeUpdateError::ValueTooLong {
19541                kind: "string",
19542                limit: MAX_SEARCH_ATTRIBUTE_STRING_LENGTH,
19543                ..
19544            })
19545        ));
19546    }
19547
19548    #[test]
19549    fn workflow_history_rejects_unpaired_or_mismatched_timer_events() {
19550        let lone_fire = WorkflowState::new(
19551            vec![history_event(
19552                "TimerFired",
19553                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19554            )],
19555            "rust-workers".to_string(),
19556            DEFAULT_CODEC.to_string(),
19557            None,
19558        )
19559        .expect_err("TimerFired requires TimerScheduled");
19560        assert!(matches!(
19561            lone_fire,
19562            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19563                if reason == "timer_schedule_missing_or_duplicate"
19564        ));
19565
19566        let wrong_identity = WorkflowState::new(
19567            vec![
19568                history_event(
19569                    "TimerScheduled",
19570                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19571                ),
19572                history_event(
19573                    "TimerFired",
19574                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
19575                ),
19576            ],
19577            "rust-workers".to_string(),
19578            DEFAULT_CODEC.to_string(),
19579            None,
19580        )
19581        .expect_err("fire must match scheduled timer identity");
19582        assert!(matches!(
19583            wrong_identity,
19584            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19585                if reason == "timer_identity_mismatch"
19586        ));
19587
19588        let duplicate_fire = WorkflowState::new(
19589            vec![
19590                history_event(
19591                    "TimerScheduled",
19592                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19593                ),
19594                history_event(
19595                    "TimerFired",
19596                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19597                ),
19598                history_event(
19599                    "TimerFired",
19600                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19601                ),
19602            ],
19603            "rust-workers".to_string(),
19604            DEFAULT_CODEC.to_string(),
19605            None,
19606        )
19607        .expect_err("a durable timer cannot fire twice");
19608        assert!(matches!(
19609            duplicate_fire,
19610            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19611                if reason == "duplicate_timer_fire"
19612        ));
19613
19614        let wrong_fired_delay = WorkflowState::new(
19615            vec![
19616                history_event(
19617                    "TimerScheduled",
19618                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19619                ),
19620                history_event(
19621                    "TimerFired",
19622                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 6}),
19623                ),
19624            ],
19625            "rust-workers".to_string(),
19626            DEFAULT_CODEC.to_string(),
19627            None,
19628        )
19629        .expect_err("timer schedule and fire delays must agree");
19630        assert!(matches!(
19631            wrong_fired_delay,
19632            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
19633                if reason == "timer_history_delay_mismatch"
19634        ));
19635    }
19636
19637    #[test]
19638    fn replay_rejects_activity_moved_before_recorded_timer() {
19639        let ctx = workflow_context(vec![
19640            history_event(
19641                "TimerScheduled",
19642                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19643            ),
19644            history_event(
19645                "TimerFired",
19646                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
19647            ),
19648            history_event(
19649                "ActivityCompleted",
19650                json!({
19651                    "sequence": 2,
19652                    "activity_type": "after-timer",
19653                    "payload_codec": DEFAULT_CODEC,
19654                    "result": fixture_envelope(json!("done")),
19655                }),
19656            ),
19657        ]);
19658        let mut activity = Box::pin(ctx.activity("after-timer", json!([])));
19659        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19660
19661        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
19662            activity.as_mut().poll(&mut task_context)
19663        else {
19664            panic!("reordered durable command must be rejected");
19665        };
19666        assert_eq!(failure.reason, "recorded_command_mismatch");
19667        assert_eq!(failure.sequence, Some(1));
19668        assert_eq!(failure.expected.as_deref(), Some("timer"));
19669        assert_eq!(failure.actual.as_deref(), Some("activity:after-timer"));
19670    }
19671
19672    #[test]
19673    fn workflow_context_emits_a_typed_named_signal_wait() {
19674        let ctx = workflow_context(Vec::new());
19675        let mut signal = Box::pin(ctx.wait_signal("finish"));
19676        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19677
19678        assert!(matches!(
19679            signal.as_mut().poll(&mut task_context),
19680            Poll::Pending
19681        ));
19682        assert_eq!(
19683            ctx.take_commands().expect("signal-wait command"),
19684            vec![json!({
19685                "type": "open_signal_wait",
19686                "signal_name": "finish",
19687            })]
19688        );
19689    }
19690
19691    #[test]
19692    fn runtime_message_stream_transport_cannot_be_opened_as_a_user_signal() {
19693        let ctx = workflow_context(Vec::new());
19694        let mut signal = Box::pin(ctx.wait_signal(MESSAGE_STREAM_SIGNAL));
19695        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19696
19697        let Poll::Ready(Err(Error::Codec(message))) = signal.as_mut().poll(&mut task_context)
19698        else {
19699            panic!("runtime-reserved signal should be rejected");
19700        };
19701        assert!(message.contains("reserved by the workflow runtime"));
19702        assert!(ctx.take_commands().expect("commands").is_empty());
19703    }
19704
19705    #[tokio::test]
19706    async fn runtime_message_stream_transport_cannot_be_sent_as_a_user_signal() {
19707        let client = Client::builder("http://127.0.0.1:9")
19708            .build()
19709            .expect("client");
19710        let error = client
19711            .signal_workflow("workflow-1", MESSAGE_STREAM_SIGNAL, json!(["forged"]))
19712            .await
19713            .expect_err("runtime-reserved signal should be rejected before transport");
19714
19715        assert!(
19716            matches!(error, Error::Codec(ref message) if message.contains("reserved by the workflow runtime"))
19717        );
19718    }
19719
19720    #[test]
19721    fn message_stream_worker_task_consumes_current_contiguous_bounded_batch() {
19722        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19723            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19724                value.to_string(),
19725            )]))
19726            .expect("message payload");
19727            json!({
19728                "schema": MESSAGE_STREAM_SCHEMA,
19729                "stream_name": "orders",
19730                "message_id": message_id,
19731                "position": position,
19732                "payload_envelope": payload,
19733            })
19734        }
19735
19736        fn opened(sequence: u64) -> HistoryEvent {
19737            history_event(
19738                "SignalWaitOpened",
19739                json!({
19740                    "sequence": sequence,
19741                    "signal_name": MESSAGE_STREAM_SIGNAL,
19742                }),
19743            )
19744        }
19745
19746        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19747            history_event(
19748                "SignalApplied",
19749                json!({
19750                    "sequence": sequence,
19751                    "signal_name": MESSAGE_STREAM_SIGNAL,
19752                    "value": fixture_envelope(json!([delivery])),
19753                }),
19754            )
19755        }
19756
19757        fn received(delivery: Value) -> HistoryEvent {
19758            history_event(
19759                "SignalReceived",
19760                json!({
19761                    "signal_name": MESSAGE_STREAM_SIGNAL,
19762                    "arguments": fixture_envelope(json!([delivery])),
19763                    "payload_codec": DEFAULT_CODEC,
19764                }),
19765            )
19766        }
19767
19768        let client = Client::new("http://127.0.0.1:8080").expect("client");
19769        let mut worker = Worker::new(client, "rust-workers");
19770        worker.register_workflow("rust.message-stream-batch", |ctx, _input| async move {
19771            let messages = ctx.message_stream("orders")?.receive(2).await?;
19772            Ok(json!(messages
19773                .into_iter()
19774                .map(|message| message.message_id)
19775                .collect::<Vec<_>>()))
19776        });
19777
19778        let first = delivery("message-1", 1, "one");
19779        let second = delivery("message-2", 2, "two");
19780        let batch = worker
19781            .execute_workflow_task_decision(workflow_task(
19782                "rust.message-stream-batch",
19783                vec![
19784                    opened(1),
19785                    received(first.clone()),
19786                    applied(1, first.clone()),
19787                    received(first.clone()),
19788                    received(second),
19789                ],
19790                DEFAULT_CODEC,
19791            ))
19792            .expect("worker task consumes the available batch");
19793
19794        assert_eq!(batch.commands.len(), 1);
19795        assert_eq!(batch.commands[0]["type"], "complete_workflow");
19796        assert_eq!(
19797            decode_wire_value(&batch.commands[0]["result"], DEFAULT_CODEC)
19798                .expect("workflow result"),
19799            json!(["message-1", "message-2"])
19800        );
19801        assert_eq!(
19802            batch.message_stream_cursors,
19803            vec![json!({"stream_name": "orders", "through_position": 2})]
19804        );
19805        assert!(batch.message_stream_waits.is_empty());
19806
19807        let partial = worker
19808            .execute_workflow_task_decision(workflow_task(
19809                "rust.message-stream-batch",
19810                vec![opened(1), received(first.clone()), applied(1, first)],
19811                DEFAULT_CODEC,
19812            ))
19813            .expect("worker task returns without waiting for a missing second item");
19814        assert_eq!(partial.commands.len(), 1);
19815        assert_eq!(partial.commands[0]["type"], "complete_workflow");
19816        assert_eq!(
19817            decode_wire_value(&partial.commands[0]["result"], DEFAULT_CODEC)
19818                .expect("workflow result"),
19819            json!(["message-1"])
19820        );
19821        assert_eq!(
19822            partial.message_stream_cursors,
19823            vec![json!({"stream_name": "orders", "through_position": 1})]
19824        );
19825        assert!(partial.message_stream_waits.is_empty());
19826    }
19827
19828    #[test]
19829    fn message_stream_replay_preserves_partial_batch_boundary_before_later_wait() {
19830        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
19831            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
19832                value.to_string(),
19833            )]))
19834            .expect("message payload");
19835            json!({
19836                "schema": MESSAGE_STREAM_SCHEMA,
19837                "stream_name": "orders",
19838                "message_id": message_id,
19839                "position": position,
19840                "payload_envelope": payload,
19841            })
19842        }
19843
19844        fn opened(sequence: u64) -> HistoryEvent {
19845            history_event(
19846                "SignalWaitOpened",
19847                json!({
19848                    "sequence": sequence,
19849                    "signal_name": MESSAGE_STREAM_SIGNAL,
19850                }),
19851            )
19852        }
19853
19854        fn received(delivery: Value) -> HistoryEvent {
19855            history_event(
19856                "SignalReceived",
19857                json!({
19858                    "signal_name": MESSAGE_STREAM_SIGNAL,
19859                    "arguments": fixture_envelope(json!([delivery])),
19860                    "payload_codec": DEFAULT_CODEC,
19861                }),
19862            )
19863        }
19864
19865        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
19866            history_event(
19867                "SignalApplied",
19868                json!({
19869                    "sequence": sequence,
19870                    "signal_name": MESSAGE_STREAM_SIGNAL,
19871                    "value": fixture_envelope(json!([delivery])),
19872                }),
19873            )
19874        }
19875
19876        let client = Client::new("http://127.0.0.1:8080").expect("client");
19877        let mut worker = Worker::new(client, "rust-workers");
19878        worker.register_workflow(
19879            "rust.message-stream-partial-batches",
19880            |ctx, _input| async move {
19881                let stream = ctx.message_stream("orders")?;
19882                let first = stream.receive(10).await?;
19883                let second = stream.receive(10).await?;
19884                Ok(json!([
19885                    first
19886                        .into_iter()
19887                        .map(|message| message.message_id)
19888                        .collect::<Vec<_>>(),
19889                    second
19890                        .into_iter()
19891                        .map(|message| message.message_id)
19892                        .collect::<Vec<_>>(),
19893                ]))
19894            },
19895        );
19896
19897        let first = delivery("message-1", 1, "one");
19898        let second = delivery("message-2", 2, "two");
19899        let decision = worker
19900            .execute_workflow_task_decision(workflow_task(
19901                "rust.message-stream-partial-batches",
19902                vec![
19903                    opened(1),
19904                    received(first.clone()),
19905                    applied(1, first),
19906                    opened(2),
19907                    received(second.clone()),
19908                    applied(2, second),
19909                ],
19910                DEFAULT_CODEC,
19911            ))
19912            .expect("cold replay preserves both authored receive boundaries");
19913
19914        assert_eq!(decision.commands.len(), 1);
19915        assert_eq!(decision.commands[0]["type"], "complete_workflow");
19916        assert_eq!(
19917            decode_wire_value(&decision.commands[0]["result"], DEFAULT_CODEC)
19918                .expect("workflow result"),
19919            json!([["message-1"], ["message-2"]])
19920        );
19921        assert_eq!(
19922            decision.message_stream_cursors,
19923            vec![json!({"stream_name": "orders", "through_position": 2})]
19924        );
19925        assert!(decision.message_stream_waits.is_empty());
19926    }
19927
19928    #[test]
19929    fn empty_message_stream_opens_internal_signal_wait_and_reports_position() {
19930        let ctx = workflow_context(Vec::new());
19931        let stream = ctx.message_stream("orders").expect("message stream");
19932        let mut receive = Box::pin(stream.receive(10));
19933        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19934
19935        assert!(matches!(
19936            receive.as_mut().poll(&mut task_context),
19937            Poll::Pending
19938        ));
19939        assert_eq!(
19940            ctx.take_commands().expect("message-stream wait command"),
19941            vec![json!({
19942                "type": "open_signal_wait",
19943                "signal_name": MESSAGE_STREAM_SIGNAL,
19944            })]
19945        );
19946        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19947        assert!(cursors.is_empty());
19948        assert_eq!(
19949            waits,
19950            vec![json!({"stream_name": "orders", "after_position": 0})]
19951        );
19952    }
19953
19954    #[test]
19955    fn continue_as_new_cursor_checkpoint_preserves_global_pending_position() {
19956        let ctx = workflow_context(vec![history_event(
19957            "SignalReceived",
19958            json!({
19959                "signal_name": MESSAGE_STREAM_SIGNAL,
19960                "arguments": fixture_envelope(json!([{
19961                    "schema": MESSAGE_STREAM_CURSOR_SCHEMA,
19962                    "stream_name": "orders",
19963                    "through_position": 2,
19964                }])),
19965                "payload_codec": DEFAULT_CODEC,
19966            }),
19967        )]);
19968        let stream = ctx.message_stream("orders").expect("message stream");
19969        let mut receive = Box::pin(stream.receive(10));
19970        let mut task_context = TaskContext::from_waker(noop_waker_ref());
19971
19972        assert!(matches!(
19973            receive.as_mut().poll(&mut task_context),
19974            Poll::Pending
19975        ));
19976        let (cursors, waits) = ctx.message_stream_metadata().expect("stream metadata");
19977        assert_eq!(
19978            cursors,
19979            vec![json!({"stream_name": "orders", "through_position": 2})]
19980        );
19981        assert_eq!(
19982            waits,
19983            vec![json!({"stream_name": "orders", "after_position": 2})]
19984        );
19985    }
19986
19987    #[test]
19988    fn message_stream_delivery_preserves_typed_avro_arguments_across_replay() {
19989        let mut empty_map = BTreeMap::new();
19990        let mut nested = BTreeMap::new();
19991        nested.insert(
19992            "value".to_string(),
19993            AvroValue::Array(vec![AvroValue::Bytes(b"nested".to_vec())]),
19994        );
19995        let values = vec![
19996            AvroValue::Bytes(vec![0, 255]),
19997            AvroValue::Long(1),
19998            AvroValue::Double(1.0),
19999            AvroValue::Array(Vec::new()),
20000            AvroValue::Map(std::mem::take(&mut empty_map)),
20001            AvroValue::Map(nested),
20002        ];
20003        let payload = encode_avro_value(&AvroValue::Array(values.clone())).expect("payload");
20004        let transport = vec![json!({
20005            "schema": MESSAGE_STREAM_SCHEMA,
20006            "stream_name": "orders",
20007            "message_id": "message-1",
20008            "position": 1,
20009            "payload_envelope": payload,
20010        })];
20011
20012        for _ in 0..2 {
20013            let Some(MessageStreamDelivery::Message(message)) =
20014                decode_message_stream_delivery(transport.clone()).expect("delivery")
20015            else {
20016                panic!("message delivery expected");
20017            };
20018            assert_eq!(message.arguments, values);
20019            assert!(matches!(message.arguments[1], AvroValue::Long(1)));
20020            assert!(matches!(message.arguments[2], AvroValue::Double(1.0)));
20021        }
20022    }
20023
20024    #[test]
20025    fn cold_worker_replacement_consumes_message_stream_wait_arrivals_once_in_order() {
20026        fn delivery(message_id: &str, position: u64, value: &str) -> Value {
20027            let payload = encode_avro_value(&AvroValue::Array(vec![AvroValue::String(
20028                value.to_string(),
20029            )]))
20030            .expect("message payload");
20031            json!({
20032                "schema": MESSAGE_STREAM_SCHEMA,
20033                "stream_name": "orders",
20034                "message_id": message_id,
20035                "position": position,
20036                "payload_envelope": payload,
20037            })
20038        }
20039
20040        fn opened(sequence: u64) -> HistoryEvent {
20041            history_event(
20042                "SignalWaitOpened",
20043                json!({
20044                    "sequence": sequence,
20045                    "signal_name": MESSAGE_STREAM_SIGNAL,
20046                }),
20047            )
20048        }
20049
20050        fn applied(sequence: u64, delivery: Value) -> HistoryEvent {
20051            history_event(
20052                "SignalApplied",
20053                json!({
20054                    "sequence": sequence,
20055                    "signal_name": MESSAGE_STREAM_SIGNAL,
20056                    "value": fixture_envelope(json!([delivery])),
20057                }),
20058            )
20059        }
20060
20061        fn worker() -> Worker {
20062            let client = Client::new("http://127.0.0.1:8080").expect("client");
20063            let mut worker = Worker::new(client, "rust-workers");
20064            worker.register_workflow("rust.message-stream", |ctx, _input| async move {
20065                let stream = ctx.message_stream("orders")?;
20066                let first = stream.receive_one().await?;
20067                let second = stream.receive_one().await?;
20068                Ok(json!([first.message_id, second.message_id]))
20069            });
20070            worker
20071        }
20072
20073        fn task_with_resume(history: Vec<HistoryEvent>, delivery: Value) -> WorkflowTask {
20074            let mut task = workflow_task("rust.message-stream", history, DEFAULT_CODEC);
20075            task.signal_name = Some(MESSAGE_STREAM_SIGNAL.to_string());
20076            task.signal_arguments = Some(fixture_envelope(json!([delivery])));
20077            task
20078        }
20079
20080        let waiting = worker()
20081            .execute_workflow_task_decision(workflow_task(
20082                "rust.message-stream",
20083                Vec::new(),
20084                DEFAULT_CODEC,
20085            ))
20086            .expect("first worker opens the stream wait");
20087        assert_eq!(
20088            waiting.commands,
20089            vec![json!({
20090                "type": "open_signal_wait",
20091                "signal_name": MESSAGE_STREAM_SIGNAL,
20092            })]
20093        );
20094        assert!(waiting.message_stream_cursors.is_empty());
20095        assert_eq!(
20096            waiting.message_stream_waits,
20097            vec![json!({"stream_name": "orders", "after_position": 0})]
20098        );
20099
20100        let first_delivery = delivery("message-1", 1, "one");
20101        let first_arrival = worker()
20102            .execute_workflow_task_decision(task_with_resume(
20103                vec![opened(1)],
20104                first_delivery.clone(),
20105            ))
20106            .expect("replacement worker consumes the first arrival");
20107        assert_eq!(
20108            first_arrival.commands,
20109            vec![json!({
20110                "type": "open_signal_wait",
20111                "signal_name": MESSAGE_STREAM_SIGNAL,
20112            })]
20113        );
20114        assert_eq!(
20115            first_arrival.message_stream_cursors,
20116            vec![json!({"stream_name": "orders", "through_position": 1})]
20117        );
20118        assert_eq!(
20119            first_arrival.message_stream_waits,
20120            vec![json!({"stream_name": "orders", "after_position": 1})]
20121        );
20122
20123        let second_delivery = delivery("message-2", 2, "two");
20124        let first_applied = applied(1, first_delivery);
20125        let completed = worker()
20126            .execute_workflow_task_decision(task_with_resume(
20127                vec![opened(1), first_applied.clone(), opened(2)],
20128                second_delivery.clone(),
20129            ))
20130            .expect("next replacement worker consumes the second arrival");
20131        assert_eq!(completed.commands.len(), 1);
20132        assert_eq!(completed.commands[0]["type"], "complete_workflow");
20133        assert_eq!(
20134            decode_wire_value(&completed.commands[0]["result"], DEFAULT_CODEC)
20135                .expect("workflow result"),
20136            json!(["message-1", "message-2"])
20137        );
20138        assert_eq!(
20139            completed.message_stream_cursors,
20140            vec![json!({"stream_name": "orders", "through_position": 2})]
20141        );
20142        assert!(completed.message_stream_waits.is_empty());
20143
20144        let replay_history = vec![
20145            opened(1),
20146            first_applied,
20147            opened(2),
20148            applied(2, second_delivery),
20149        ];
20150        for _cold_worker_or_restart in 0..2 {
20151            let replayed = worker()
20152                .execute_workflow_task_decision(workflow_task(
20153                    "rust.message-stream",
20154                    replay_history.clone(),
20155                    DEFAULT_CODEC,
20156                ))
20157                .expect("cold worker replays each logical message exactly once");
20158            assert_eq!(replayed.commands.len(), 1);
20159            assert_eq!(
20160                decode_wire_value(&replayed.commands[0]["result"], DEFAULT_CODEC)
20161                    .expect("replayed workflow result"),
20162                json!(["message-1", "message-2"])
20163            );
20164            assert_eq!(
20165                replayed.message_stream_cursors,
20166                vec![json!({"stream_name": "orders", "through_position": 2})]
20167            );
20168            assert!(replayed.message_stream_waits.is_empty());
20169        }
20170    }
20171
20172    #[test]
20173    fn message_stream_capability_and_completion_require_protocol_one_fifteen() {
20174        assert!(!worker_protocol_supports_message_streams("1.14"));
20175        assert!(worker_protocol_supports_message_streams("1.15"));
20176        assert!(worker_protocol_supports_message_streams("1.16"));
20177        assert!(worker_protocol_supports_message_streams(
20178            WORKER_PROTOCOL_VERSION
20179        ));
20180        assert_eq!(MESSAGE_STREAMS_MINIMUM_WORKER_PROTOCOL_VERSION, "1.15");
20181    }
20182
20183    #[test]
20184    fn condition_wait_history_cannot_be_consumed_as_a_typed_signal_wait() {
20185        let ctx = workflow_context(vec![
20186            history_event(
20187                "ConditionWaitOpened",
20188                json!({
20189                    "sequence": 1,
20190                    "condition_wait_id": "condition:1",
20191                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20192                    "condition_key": "signal:finish",
20193                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20194                }),
20195            ),
20196            history_event(
20197                "ConditionWaitSatisfied",
20198                json!({
20199                    "sequence": 1,
20200                    "condition_wait_id": "condition:1",
20201                    "condition_wait_occurrence_id": "rust:condition-wait:0",
20202                    "condition_key": "signal:finish",
20203                    "condition_definition_fingerprint": "sha256:signal-finish-v1",
20204                }),
20205            ),
20206            history_event(
20207                "SignalReceived",
20208                json!({"signal_name": "finish", "arguments": []}),
20209            ),
20210        ]);
20211        let mut signal = Box::pin(ctx.wait_signal("finish"));
20212        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20213
20214        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20215            signal.as_mut().poll(&mut task_context)
20216        else {
20217            panic!("condition history must not resolve as a typed signal wait");
20218        };
20219        assert_eq!(failure.reason, "recorded_command_mismatch");
20220        assert_eq!(failure.expected.as_deref(), Some("condition wait"));
20221    }
20222
20223    #[test]
20224    fn replay_orders_signal_waits_and_timers_in_one_command_stream() {
20225        let signal_then_timer = vec![
20226            history_event(
20227                "SignalWaitOpened",
20228                json!({"sequence": 1, "signal_name": "go"}),
20229            ),
20230            history_event(
20231                "SignalApplied",
20232                json!({
20233                    "sequence": 1,
20234                    "signal_name": "go",
20235                    "value": fixture_envelope(json!(["now"])),
20236                }),
20237            ),
20238            history_event(
20239                "TimerScheduled",
20240                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20241            ),
20242            history_event(
20243                "TimerFired",
20244                json!({"sequence": 2, "timer_id": "timer-2", "delay_seconds": 5}),
20245            ),
20246        ];
20247
20248        let ctx = workflow_context(signal_then_timer.clone());
20249        let mut signal = Box::pin(ctx.wait_signal("go"));
20250        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20251        assert!(matches!(
20252            signal.as_mut().poll(&mut task_context),
20253            Poll::Ready(Ok(arguments)) if arguments == vec![json!("now")]
20254        ));
20255        let mut timer = Box::pin(ctx.sleep(Duration::from_secs(5)));
20256        assert!(matches!(
20257            timer.as_mut().poll(&mut task_context),
20258            Poll::Ready(Ok(()))
20259        ));
20260        ctx.ensure_history_consumed()
20261            .expect("signal and timer history consumed in order");
20262
20263        let reordered = workflow_context(signal_then_timer);
20264        let mut timer_first = Box::pin(reordered.sleep(Duration::from_secs(5)));
20265        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20266            timer_first.as_mut().poll(&mut task_context)
20267        else {
20268            panic!("timer cannot consume signal-wait-first history");
20269        };
20270        assert_eq!(failure.reason, "recorded_command_mismatch");
20271        assert_eq!(failure.sequence, Some(1));
20272        assert_eq!(failure.expected.as_deref(), Some("signal wait"));
20273
20274        let timer_then_signal = vec![
20275            history_event(
20276                "TimerScheduled",
20277                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20278            ),
20279            history_event(
20280                "TimerFired",
20281                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20282            ),
20283            history_event(
20284                "SignalWaitOpened",
20285                json!({"sequence": 2, "signal_name": "go"}),
20286            ),
20287            history_event(
20288                "SignalApplied",
20289                json!({
20290                    "sequence": 2,
20291                    "signal_name": "go",
20292                    "value": fixture_envelope(json!([])),
20293                }),
20294            ),
20295        ];
20296        let reordered = workflow_context(timer_then_signal);
20297        let mut signal_first = Box::pin(reordered.wait_signal("go"));
20298        let Poll::Ready(Err(Error::NonDeterministicReplay(failure))) =
20299            signal_first.as_mut().poll(&mut task_context)
20300        else {
20301            panic!("signal wait cannot consume timer-first history");
20302        };
20303        assert_eq!(failure.reason, "recorded_command_mismatch");
20304        assert_eq!(failure.sequence, Some(1));
20305        assert_eq!(failure.expected.as_deref(), Some("timer"));
20306    }
20307
20308    #[test]
20309    fn workflow_history_rejects_duplicate_or_colliding_command_sequences() {
20310        let duplicate_timer = WorkflowState::new(
20311            vec![
20312                history_event(
20313                    "TimerScheduled",
20314                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20315                ),
20316                history_event(
20317                    "TimerScheduled",
20318                    json!({"sequence": 1, "timer_id": "timer-2", "delay_seconds": 5}),
20319                ),
20320            ],
20321            "rust-workers".to_string(),
20322            DEFAULT_CODEC.to_string(),
20323            None,
20324        )
20325        .expect_err("one workflow sequence cannot schedule two timers");
20326        assert!(matches!(
20327            duplicate_timer,
20328            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20329                if reason == "timer_schedule_missing_or_duplicate"
20330        ));
20331
20332        let colliding_kinds = WorkflowState::new(
20333            vec![
20334                history_event(
20335                    "TimerScheduled",
20336                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20337                ),
20338                history_event(
20339                    "ActivityCompleted",
20340                    json!({"sequence": 1, "activity_type": "same-sequence"}),
20341                ),
20342            ],
20343            "rust-workers".to_string(),
20344            DEFAULT_CODEC.to_string(),
20345            None,
20346        )
20347        .expect_err("one workflow sequence cannot identify two command kinds");
20348        assert!(matches!(
20349            colliding_kinds,
20350            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20351                if reason == "durable_command_sequence_collision"
20352        ));
20353
20354        let duplicate_signal_wait = WorkflowState::new(
20355            vec![
20356                history_event(
20357                    "SignalWaitOpened",
20358                    json!({"sequence": 1, "signal_name": "go"}),
20359                ),
20360                history_event(
20361                    "SignalWaitOpened",
20362                    json!({"sequence": 1, "signal_name": "go"}),
20363                ),
20364            ],
20365            "rust-workers".to_string(),
20366            DEFAULT_CODEC.to_string(),
20367            None,
20368        )
20369        .expect_err("one workflow sequence cannot open two signal waits");
20370        assert!(matches!(
20371            duplicate_signal_wait,
20372            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20373                if reason == "signal_wait_open_missing_or_duplicate"
20374        ));
20375    }
20376
20377    #[test]
20378    fn workflow_history_accepts_a_first_command_after_global_sequence_gaps() {
20379        let result = encode_value_envelope(&json!({"captured": true}), DEFAULT_CODEC)
20380            .expect("side-effect result");
20381        let ctx = workflow_context(vec![history_event(
20382            "SideEffectRecorded",
20383            json!({"sequence": 99, "result": result}),
20384        )]);
20385
20386        let replayed: Value = ctx
20387            .side_effect(|| panic!("recorded side effect must not run"))
20388            .expect("positive global workflow sequence is valid");
20389        assert_eq!(replayed, json!({"captured": true}));
20390        ctx.ensure_history_consumed().expect("history consumed");
20391    }
20392
20393    #[test]
20394    fn workflow_history_rejects_zero_and_descending_command_sequences() {
20395        let result =
20396            encode_value_envelope(&json!("captured"), DEFAULT_CODEC).expect("side-effect result");
20397        let zero = WorkflowState::new(
20398            vec![history_event(
20399                "SideEffectRecorded",
20400                json!({"sequence": 0, "result": result.clone()}),
20401            )],
20402            "rust-workers".to_string(),
20403            DEFAULT_CODEC.to_string(),
20404            None,
20405        )
20406        .expect_err("durable command sequences must be positive");
20407        assert!(matches!(
20408            zero,
20409            Error::NonDeterministicReplay(ReplayFailure { ref reason, .. })
20410                if reason == "durable_command_sequence_invalid"
20411        ));
20412
20413        let descending = WorkflowState::new(
20414            vec![
20415                history_event(
20416                    "SideEffectRecorded",
20417                    json!({"sequence": 3, "result": result}),
20418                ),
20419                history_event(
20420                    "VersionMarkerRecorded",
20421                    json!({
20422                        "sequence": 2,
20423                        "change_id": "descending-marker",
20424                        "version": 1,
20425                        "min_supported": 1,
20426                        "max_supported": 1,
20427                    }),
20428                ),
20429            ],
20430            "rust-workers".to_string(),
20431            DEFAULT_CODEC.to_string(),
20432            None,
20433        )
20434        .expect_err("new durable commands must remain strictly ordered");
20435        let Error::NonDeterministicReplay(failure) = descending else {
20436            panic!("expected typed replay failure");
20437        };
20438        assert_eq!(failure.reason, "durable_command_sequence_mismatch");
20439        assert_eq!(failure.sequence, Some(2));
20440        assert_eq!(
20441            failure.expected.as_deref(),
20442            Some("workflow sequence greater than 3")
20443        );
20444        assert_eq!(failure.actual.as_deref(), Some("2"));
20445    }
20446
20447    #[test]
20448    fn workflow_task_replay_completes_after_signals_create_sequence_gaps() {
20449        fn worker() -> Worker {
20450            let client = Client::new("http://127.0.0.1:8080").expect("client");
20451            let mut worker = Worker::new(client, "rust-workers");
20452            worker.register_workflow("rust.finish-after-gaps", |ctx, _input| async move {
20453                ctx.wait_signal("finish").await?;
20454                let marker: String =
20455                    ctx.side_effect(|| panic!("recorded side effect must not run"))?;
20456                assert_eq!(marker, "after-finish");
20457                Ok(json!("finished"))
20458            });
20459            worker
20460        }
20461
20462        let marker = encode_value_envelope(&json!("after-finish"), DEFAULT_CODEC)
20463            .expect("side-effect result");
20464        let task = workflow_task(
20465            "rust.finish-after-gaps",
20466            vec![
20467                history_event(
20468                    "SignalWaitOpened",
20469                    json!({"sequence": 1, "signal_name": "finish"}),
20470                ),
20471                history_event(
20472                    "SignalReceived",
20473                    json!({
20474                        "signal_id": "increment-3",
20475                        "signal_name": "increment",
20476                        "workflow_sequence": 2,
20477                        "payload_codec": DEFAULT_CODEC,
20478                        "arguments": fixture_envelope(json!([3])),
20479                    }),
20480                ),
20481                history_event(
20482                    "SignalReceived",
20483                    json!({
20484                        "signal_id": "increment-5",
20485                        "signal_name": "increment",
20486                        "workflow_sequence": 3,
20487                        "payload_codec": DEFAULT_CODEC,
20488                        "arguments": fixture_envelope(json!([5])),
20489                    }),
20490                ),
20491                history_event(
20492                    "SignalReceived",
20493                    json!({
20494                        "signal_id": "finish",
20495                        "signal_name": "finish",
20496                        "workflow_sequence": 4,
20497                        "payload_codec": DEFAULT_CODEC,
20498                        "arguments": fixture_envelope(json!([])),
20499                    }),
20500                ),
20501                history_event(
20502                    "SignalApplied",
20503                    json!({
20504                        "sequence": 1,
20505                        "signal_id": "finish",
20506                        "signal_name": "finish",
20507                        "payload_codec": DEFAULT_CODEC,
20508                        "value": fixture_envelope(json!([])),
20509                    }),
20510                ),
20511                history_event(
20512                    "SideEffectRecorded",
20513                    json!({"sequence": 5, "result": marker}),
20514                ),
20515            ],
20516            DEFAULT_CODEC,
20517        );
20518
20519        for _original_or_cold_worker in 0..2 {
20520            let commands = worker()
20521                .execute_workflow_task(task.clone())
20522                .expect("signal gaps preserve deterministic replay");
20523            assert_eq!(commands.len(), 1, "replay emits only terminal completion");
20524            assert_eq!(commands[0]["type"], "complete_workflow");
20525            assert_eq!(
20526                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("workflow output"),
20527                json!("finished")
20528            );
20529        }
20530    }
20531
20532    #[test]
20533    fn workflow_sleep_rejects_unrepresentable_rounded_duration() {
20534        let ctx = workflow_context(Vec::new());
20535        let mut sleep = Box::pin(ctx.start_timer(Duration::new(u64::MAX, 1)));
20536        let mut task_context = TaskContext::from_waker(noop_waker_ref());
20537        assert!(matches!(
20538            sleep.as_mut().poll(&mut task_context),
20539            Poll::Ready(Err(Error::TimerDurationOverflow))
20540        ));
20541        assert!(ctx.take_commands().expect("commands").is_empty());
20542    }
20543
20544    #[test]
20545    fn workflow_memo_update_emits_canonical_command_and_replays_once() {
20546        let entries = AvroValue::Map(BTreeMap::from([
20547            ("text".to_string(), AvroValue::String("same".to_string())),
20548            (
20549                "nested".to_string(),
20550                AvroValue::Map(BTreeMap::from([
20551                    ("beta".to_string(), AvroValue::Long(2)),
20552                    ("alpha".to_string(), AvroValue::Long(1)),
20553                ])),
20554            ),
20555            ("long".to_string(), AvroValue::Long(7)),
20556            ("double".to_string(), AvroValue::Double(7.0)),
20557            ("binary".to_string(), AvroValue::Bytes(b"same".to_vec())),
20558        ]));
20559        let ctx = workflow_context(Vec::new());
20560        ctx.upsert_memo(entries.clone()).expect("valid memo update");
20561        let commands = ctx.take_commands().expect("commands");
20562
20563        assert_eq!(commands.len(), 1);
20564        assert_eq!(commands[0]["type"], "upsert_memo");
20565        let server_entries = json!({
20566            "codec": "avro",
20567            "blob": "wwHioz3/VYAiNw4KDGJpbmFyeQgIc2FtZQxkb3VibGUGAAAAAAAAHEAIbG9uZwQODG5lc3RlZA4ECmFscGhhBAIIYmV0YQQEAAh0ZXh0CghzYW1lAA==",
20568        });
20569        assert_eq!(
20570            commands[0]["entries"]
20571                .as_object()
20572                .expect("entries envelope")
20573                .keys()
20574                .collect::<Vec<_>>(),
20575            vec!["blob", "codec"]
20576        );
20577        assert_eq!(commands[0]["entries"], server_entries);
20578        let wire_entries =
20579            decode_wire_avro_value(&commands[0]["entries"], DEFAULT_CODEC).expect("memo entries");
20580        assert_eq!(wire_entries, entries);
20581
20582        let history = vec![history_event(
20583            "MemoUpserted",
20584            json!({
20585                "sequence": 1,
20586                "entries": server_entries.clone(),
20587                "merged": server_entries,
20588            }),
20589        )];
20590        let replay = workflow_context(history.clone());
20591        replay
20592            .upsert_memo(entries.clone())
20593            .expect("matching replay identity");
20594        assert!(replay.take_commands().expect("replay commands").is_empty());
20595
20596        let changed_types = AvroValue::Map(BTreeMap::from([
20597            ("text".to_string(), AvroValue::Bytes(b"same".to_vec())),
20598            (
20599                "nested".to_string(),
20600                AvroValue::Map(BTreeMap::from([
20601                    ("alpha".to_string(), AvroValue::Long(1)),
20602                    ("beta".to_string(), AvroValue::Long(2)),
20603                ])),
20604            ),
20605            ("long".to_string(), AvroValue::Double(7.0)),
20606            ("double".to_string(), AvroValue::Long(7)),
20607            ("binary".to_string(), AvroValue::String("same".to_string())),
20608        ]));
20609        let error = workflow_context(history)
20610            .upsert_memo(changed_types)
20611            .expect_err("memo replay identity must preserve Avro value types");
20612        assert!(matches!(
20613            error,
20614            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20615        ));
20616    }
20617
20618    #[test]
20619    fn workflow_memo_update_rejects_changed_replay_identity_and_invalid_keys() {
20620        let original = encode_value_envelope(&json!({"stage": "original"}), DEFAULT_CODEC)
20621            .expect("memo envelope");
20622        let replay = workflow_context(vec![history_event(
20623            "MemoUpserted",
20624            json!({
20625                "sequence": 1,
20626                "entries": original.clone(),
20627                "merged": original
20628            }),
20629        )]);
20630        let error = replay
20631            .upsert_memo(json!({"stage": "changed"}))
20632            .expect_err("changed memo update must fail replay");
20633        assert!(matches!(
20634            error,
20635            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20636        ));
20637
20638        let invalid = workflow_context(Vec::new())
20639            .upsert_memo(
20640                json!({"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx": true}),
20641            )
20642            .expect_err("oversized key");
20643        assert!(matches!(invalid, Error::InvalidMemoUpdate(_)));
20644    }
20645
20646    #[test]
20647    fn workflow_memo_replay_distinguishes_signed_zero_identity() {
20648        let negative_zero = AvroValue::Map(BTreeMap::from([(
20649            "reading".to_string(),
20650            AvroValue::Double(-0.0),
20651        )]));
20652        let negative_zero_envelope =
20653            encode_typed_envelope(&negative_zero, DEFAULT_CODEC).expect("negative zero envelope");
20654        let history = vec![history_event(
20655            "MemoUpserted",
20656            json!({
20657                "sequence": 1,
20658                "entries": negative_zero_envelope.clone(),
20659                "merged": negative_zero_envelope,
20660            }),
20661        )];
20662
20663        workflow_context(history.clone())
20664            .upsert_memo(negative_zero)
20665            .expect("matching negative-zero history identity");
20666
20667        let error = workflow_context(history)
20668            .upsert_memo(AvroValue::Map(BTreeMap::from([(
20669                "reading".to_string(),
20670                AvroValue::Double(0.0),
20671            )])))
20672            .expect_err("positive zero must not consume negative-zero memo history");
20673        assert!(matches!(
20674            error,
20675            Error::NonDeterministicReplay(ref failure) if failure.reason == "memo_update_mismatch"
20676        ));
20677    }
20678
20679    #[test]
20680    fn workflow_memo_capability_requires_flag_and_command_advertisement() {
20681        let supported = json!({
20682            "workflow_memo_updates": {"supported": true, "minimum_protocol_version": "1.14"},
20683            "supported_workflow_task_commands": ["complete_workflow", "upsert_memo"]
20684        });
20685        assert!(runtime_supports_workflow_memo_updates(Some(&supported)));
20686        assert!(!runtime_supports_workflow_memo_updates(Some(&json!({
20687            "workflow_memo_updates": {"supported": false},
20688            "supported_workflow_task_commands": ["upsert_memo"]
20689        }))));
20690        assert!(commands_use_workflow_memo_updates(&[json!({
20691            "type": "upsert_memo",
20692            "entries": {"stage": "processing"}
20693        })]));
20694    }
20695
20696    #[test]
20697    fn workflow_task_replay_completes_without_rescheduling_recorded_commands() {
20698        let client = Client::new("http://127.0.0.1:8080").expect("client");
20699        let mut worker = Worker::new(client, "rust-workers");
20700        worker.register_workflow("rust.timer", |ctx, _input| async move {
20701            ctx.sleep(Duration::from_secs(5)).await?;
20702            ctx.activity("after-timer", json!([])).await
20703        });
20704
20705        let task = |history_events| WorkflowTask {
20706            task_id: "wft-rust-timer-1".to_string(),
20707            workflow_command_id: None,
20708            workflow_id: Some("wf-rust-timer".to_string()),
20709            run_id: Some("run-rust-timer".to_string()),
20710            workflow_type: "rust.timer".to_string(),
20711            cancel_requested: false,
20712            payload_codec: DEFAULT_CODEC.to_string(),
20713            arguments: Some(
20714                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
20715            ),
20716            history_events,
20717            total_history_events: None,
20718            history_size_bytes: None,
20719            continue_as_new_recommended: None,
20720            history_budget_pressure: None,
20721            next_history_page_token: None,
20722            workflow_task_attempt: 1,
20723            workflow_signal_id: None,
20724            signal_name: None,
20725            signal_arguments: None,
20726            workflow_update_id: None,
20727            update_name: None,
20728            lease_owner: Some("rust-worker".to_string()),
20729        };
20730
20731        let initial = worker
20732            .execute_workflow_task(task(Vec::new()))
20733            .expect("initial timer task");
20734        assert_eq!(
20735            initial,
20736            vec![json!({"type": "start_timer", "delay_seconds": 5})]
20737        );
20738
20739        let activity_result =
20740            encode_value_envelope(&json!("done"), DEFAULT_CODEC).expect("activity result");
20741        let replayed = worker
20742            .execute_workflow_task(task(vec![
20743                history_event(
20744                    "TimerScheduled",
20745                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20746                ),
20747                history_event(
20748                    "TimerFired",
20749                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
20750                ),
20751                history_event(
20752                    "ActivityCompleted",
20753                    json!({
20754                        "sequence": 2,
20755                        "activity_type": "after-timer",
20756                        "payload_codec": DEFAULT_CODEC,
20757                        "result": activity_result,
20758                    }),
20759                ),
20760            ]))
20761            .expect("replayed workflow task");
20762        assert_eq!(replayed.len(), 1);
20763        assert_eq!(replayed[0]["type"], "complete_workflow");
20764        assert_eq!(
20765            decode_wire_value(&replayed[0]["result"], DEFAULT_CODEC).expect("result"),
20766            json!("done")
20767        );
20768    }
20769
20770    #[test]
20771    fn workflow_continue_as_new_emits_arguments_type_and_queue_once() {
20772        let client = Client::new("http://127.0.0.1:8080").expect("client");
20773        let mut worker = Worker::new(client, "rust-workers");
20774        worker.register_workflow("rust.continue", |ctx, _input| async move {
20775            ctx.continue_as_new_with_options(
20776                ContinueAsNewOptions::new()
20777                    .workflow_type("rust.next")
20778                    .task_queue("next-workers"),
20779                json!([2, {"cursor": "next"}]),
20780            )
20781        });
20782
20783        let commands = worker
20784            .execute_workflow_task(workflow_task("rust.continue", Vec::new(), DEFAULT_CODEC))
20785            .expect("continue-as-new command");
20786
20787        assert_eq!(commands.len(), 1);
20788        assert_eq!(commands[0]["type"], "continue_as_new");
20789        assert_eq!(commands[0]["workflow_type"], "rust.next");
20790        assert_eq!(commands[0]["queue"], "next-workers");
20791        assert_eq!(
20792            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC)
20793                .expect("continue-as-new arguments"),
20794            json!([2, {"cursor": "next"}])
20795        );
20796    }
20797
20798    #[test]
20799    fn continue_as_new_preserves_typed_arguments() {
20800        let client = Client::new("http://127.0.0.1:8080").expect("client");
20801        let mut worker = Worker::new(client, "rust-workers");
20802        worker.register_workflow_avro_value("rust.typed-continue", |ctx, _input| async move {
20803            ctx.continue_as_new(AvroValue::Array(vec![typed_fidelity_probe()]))?;
20804            unreachable!("continue-as-new returns a control-flow error")
20805        });
20806
20807        let commands = worker
20808            .execute_workflow_task(workflow_task(
20809                "rust.typed-continue",
20810                Vec::new(),
20811                DEFAULT_CODEC,
20812            ))
20813            .expect("typed continue-as-new command");
20814
20815        assert_eq!(commands[0]["type"], "continue_as_new");
20816        assert_eq!(
20817            decode_wire_avro_value(&commands[0]["arguments"], DEFAULT_CODEC)
20818                .expect("typed continue arguments"),
20819            AvroValue::Array(vec![typed_fidelity_probe()])
20820        );
20821    }
20822
20823    #[test]
20824    fn recorded_continue_as_new_is_consumed_without_duplicate_successor_command() {
20825        let client = Client::new("http://127.0.0.1:8080").expect("client");
20826        let mut worker = Worker::new(client, "rust-workers");
20827        worker.register_workflow("rust.continue", |ctx, _input| async move {
20828            ctx.continue_as_new(json!([2]))
20829        });
20830        let task = workflow_task(
20831            "rust.continue",
20832            vec![history_event(
20833                "WorkflowContinuedAsNew",
20834                json!({"sequence": 1, "continued_to_run_id": "run-next"}),
20835            )],
20836            DEFAULT_CODEC,
20837        );
20838
20839        for _worker_restart_or_redelivery in 0..2 {
20840            let commands = worker
20841                .execute_workflow_task(task.clone())
20842                .expect("recorded transition replays");
20843            assert!(
20844                commands.is_empty(),
20845                "replay must not emit another successor"
20846            );
20847        }
20848    }
20849
20850    #[test]
20851    fn continue_as_new_rejects_invalid_overrides_before_emitting_a_command() {
20852        let ctx = workflow_context(Vec::new());
20853        let error = ctx
20854            .continue_as_new_with_options(ContinueAsNewOptions::new().task_queue("  "), json!([1]))
20855            .expect_err("blank queue must be rejected");
20856
20857        let Error::InvalidContinueAsNewOptions(error) = error else {
20858            panic!("expected typed continue-as-new validation error");
20859        };
20860        assert_eq!(error.field, "task_queue");
20861        assert!(ctx.take_commands().expect("commands").is_empty());
20862    }
20863
20864    #[test]
20865    fn workflow_context_exposes_server_history_budget() {
20866        let client = Client::new("http://127.0.0.1:8080").expect("client");
20867        let mut worker = Worker::new(client, "rust-workers");
20868        worker.register_workflow("rust.history-budget", |ctx, _input| async move {
20869            let budget = ctx.history_budget()?;
20870            Ok(json!({
20871                "events": budget.event_count,
20872                "bytes": budget.size_bytes,
20873                "recommended": budget.continue_as_new_recommended,
20874                "pressure": budget.pressure,
20875            }))
20876        });
20877        let task: WorkflowTask = serde_json::from_value(json!({
20878            "task_id": "task-history-budget",
20879            "workflow_type": "rust.history-budget",
20880            "payload_codec": DEFAULT_CODEC,
20881            "history_events": [],
20882            "total_history_events": 480,
20883            "history_size_bytes": 1_048_576,
20884            "continue_as_new_recommended": true,
20885            "history_budget_pressure": "continue_as_new_recommended",
20886        }))
20887        .expect("published workflow task");
20888
20889        let commands = worker
20890            .execute_workflow_task(task)
20891            .expect("history-budget workflow");
20892        let result = decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("result");
20893        assert_eq!(result["events"], 480);
20894        assert_eq!(result["bytes"], 1_048_576);
20895        assert_eq!(result["recommended"], true);
20896        assert_eq!(result["pressure"], "continue_as_new_recommended");
20897    }
20898
20899    #[test]
20900    fn uncaught_workflow_handler_error_emits_terminal_failure_command() {
20901        let client = Client::new("http://127.0.0.1:8080").expect("client");
20902        let mut worker = Worker::new(client, "rust-workers");
20903        worker.register_workflow("rust.failing", |_ctx, _input| async move {
20904            Err(Error::Codec("rust_conformance_failure".to_string()))
20905        });
20906        let task = WorkflowTask {
20907            task_id: "wft-rust-failing-1".to_string(),
20908            workflow_command_id: None,
20909            workflow_id: Some("wf-rust-failing".to_string()),
20910            run_id: Some("run-rust-failing".to_string()),
20911            workflow_type: "rust.failing".to_string(),
20912            cancel_requested: false,
20913            payload_codec: DEFAULT_CODEC.to_string(),
20914            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
20915            history_events: Vec::new(),
20916            total_history_events: Some(0),
20917            history_size_bytes: None,
20918            continue_as_new_recommended: None,
20919            history_budget_pressure: None,
20920            next_history_page_token: None,
20921            workflow_task_attempt: 1,
20922            workflow_signal_id: None,
20923            signal_name: None,
20924            signal_arguments: None,
20925            workflow_update_id: None,
20926            update_name: None,
20927            lease_owner: Some("rust-worker".to_string()),
20928        };
20929
20930        let commands = worker
20931            .execute_workflow_task(task)
20932            .expect("handler failure becomes a workflow command");
20933
20934        assert_eq!(commands.len(), 1);
20935        assert_eq!(commands[0]["type"], "fail_workflow");
20936        assert_eq!(commands[0]["exception_type"], "RustWorkflowError");
20937        assert_eq!(commands[0]["exception_class"], "durable_workflow::Error");
20938        assert_eq!(commands[0]["non_retryable"], false);
20939        assert_eq!(
20940            commands[0]["message"],
20941            "codec error: rust_conformance_failure"
20942        );
20943        assert_eq!(
20944            commands[0]["exception"]["message"],
20945            "codec error: rust_conformance_failure"
20946        );
20947    }
20948
20949    #[test]
20950    fn ordinary_handler_error_preserves_commands_queued_in_the_same_decision() {
20951        let client = Client::new("http://127.0.0.1:8080").expect("client");
20952        let mut worker = Worker::new(client, "rust-workers");
20953        worker.register_workflow("rust.failing-after-side-effect", |ctx, _input| async move {
20954            let _: String = ctx.side_effect(|| "captured".to_string())?;
20955            Err(Error::WorkerLoop("application failure".to_string()))
20956        });
20957
20958        let commands = worker
20959            .execute_workflow_task(workflow_task(
20960                "rust.failing-after-side-effect",
20961                Vec::new(),
20962                DEFAULT_CODEC,
20963            ))
20964            .expect("ordinary failure remains a workflow decision");
20965
20966        assert_eq!(commands.len(), 2);
20967        assert_eq!(commands[0]["type"], "record_side_effect");
20968        assert_eq!(commands[1]["type"], "fail_workflow");
20969    }
20970
20971    #[test]
20972    fn handler_error_cannot_hide_an_unconsumed_committed_side_effect() {
20973        let client = Client::new("http://127.0.0.1:8080").expect("client");
20974        let mut worker = Worker::new(client, "rust-workers");
20975        worker.register_workflow("rust.removed-side-effect", |_ctx, _input| async move {
20976            Err(Error::WorkerLoop("application failure".to_string()))
20977        });
20978        let result =
20979            encode_value_envelope(&json!("committed"), DEFAULT_CODEC).expect("side-effect result");
20980
20981        let error = worker
20982            .execute_workflow_task(workflow_task(
20983                "rust.removed-side-effect",
20984                vec![history_event(
20985                    "SideEffectRecorded",
20986                    json!({"sequence": 1, "result": result}),
20987                )],
20988                DEFAULT_CODEC,
20989            ))
20990            .expect_err("removed committed history must not become fail_workflow");
20991
20992        let Error::NonDeterministicReplay(failure) = error else {
20993            panic!("expected typed replay failure");
20994        };
20995        assert_eq!(failure.reason, "recorded_commands_unconsumed");
20996        assert_eq!(failure.sequence, Some(1));
20997        assert_eq!(failure.expected.as_deref(), Some("side effect"));
20998    }
20999
21000    #[test]
21001    fn replay_error_discards_side_effect_queued_before_incompatible_marker_check() {
21002        let client = Client::new("http://127.0.0.1:8080").expect("client");
21003        let mut worker = Worker::new(client, "rust-workers");
21004        worker.register_workflow(
21005            "rust.side-effect-before-marker-error",
21006            |ctx, _input| async move {
21007                assert_eq!(ctx.get_version("restart-safe", 1, 1)?, 1);
21008                let _: String = ctx.side_effect(|| "must-not-commit".to_string())?;
21009                ctx.get_version("restart-safe", 2, 2)?;
21010                Ok(Value::Null)
21011            },
21012        );
21013
21014        let error = worker
21015            .execute_workflow_task(workflow_task(
21016                "rust.side-effect-before-marker-error",
21017                vec![history_event(
21018                    "VersionMarkerRecorded",
21019                    json!({
21020                        "sequence": 1,
21021                        "change_id": "restart-safe",
21022                        "version": 1,
21023                        "min_supported": 1,
21024                        "max_supported": 1,
21025                    }),
21026                )],
21027                DEFAULT_CODEC,
21028            ))
21029            .expect_err("replay error must return no queued workflow commands");
21030
21031        let Error::NonDeterministicReplay(failure) = error else {
21032            panic!("expected typed replay failure");
21033        };
21034        assert_eq!(failure.reason, "version_marker_incompatible_range");
21035        assert_eq!(failure.sequence, Some(1));
21036    }
21037
21038    #[test]
21039    fn workflow_task_replay_keeps_recorded_unfired_timer_pending_without_rescheduling() {
21040        let client = Client::new("http://127.0.0.1:8080").expect("client");
21041        let mut worker = Worker::new(client, "rust-workers");
21042        worker.register_workflow("rust.timer.pending", |ctx, _input| async move {
21043            ctx.sleep(Duration::from_secs(5)).await?;
21044            Ok(json!({"status": "timer fired"}))
21045        });
21046
21047        let task = WorkflowTask {
21048            task_id: "wft-rust-timer-pending".to_string(),
21049            workflow_command_id: None,
21050            workflow_id: Some("wf-rust-timer".to_string()),
21051            run_id: Some("run-rust-timer".to_string()),
21052            workflow_type: "rust.timer.pending".to_string(),
21053            cancel_requested: false,
21054            payload_codec: DEFAULT_CODEC.to_string(),
21055            arguments: Some(
21056                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21057            ),
21058            history_events: vec![history_event(
21059                "TimerScheduled",
21060                json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21061            )],
21062            total_history_events: Some(1),
21063            history_size_bytes: None,
21064            continue_as_new_recommended: None,
21065            history_budget_pressure: None,
21066            next_history_page_token: None,
21067            workflow_task_attempt: 1,
21068            workflow_signal_id: None,
21069            signal_name: None,
21070            signal_arguments: None,
21071            workflow_update_id: None,
21072            update_name: None,
21073            lease_owner: Some("rust-worker".to_string()),
21074        };
21075
21076        for _redelivery_or_restart in 0..2 {
21077            let commands = worker
21078                .execute_workflow_task(task.clone())
21079                .expect("recorded timer remains pending");
21080            assert!(
21081                commands.is_empty(),
21082                "recorded timer must not be rescheduled"
21083            );
21084        }
21085    }
21086
21087    #[test]
21088    fn workflow_task_rejects_recorded_command_removed_from_workflow_code() {
21089        let client = Client::new("http://127.0.0.1:8080").expect("client");
21090        let mut worker = Worker::new(client, "rust-workers");
21091        worker.register_workflow("rust.timer.removed", |_ctx, _input| async move {
21092            Ok(json!({"status": "completed"}))
21093        });
21094        let task = WorkflowTask {
21095            task_id: "wft-rust-timer-removed".to_string(),
21096            workflow_command_id: None,
21097            workflow_id: Some("wf-rust-timer".to_string()),
21098            run_id: Some("run-rust-timer".to_string()),
21099            workflow_type: "rust.timer.removed".to_string(),
21100            cancel_requested: false,
21101            payload_codec: DEFAULT_CODEC.to_string(),
21102            arguments: Some(
21103                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21104            ),
21105            history_events: vec![
21106                history_event(
21107                    "TimerScheduled",
21108                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21109                ),
21110                history_event(
21111                    "TimerFired",
21112                    json!({"sequence": 1, "timer_id": "timer-1", "delay_seconds": 5}),
21113                ),
21114            ],
21115            total_history_events: Some(2),
21116            history_size_bytes: None,
21117            continue_as_new_recommended: None,
21118            history_budget_pressure: None,
21119            next_history_page_token: None,
21120            workflow_task_attempt: 1,
21121            workflow_signal_id: None,
21122            signal_name: None,
21123            signal_arguments: None,
21124            workflow_update_id: None,
21125            update_name: None,
21126            lease_owner: Some("rust-worker".to_string()),
21127        };
21128
21129        let Error::NonDeterministicReplay(failure) = worker
21130            .execute_workflow_task(task)
21131            .expect_err("removed timer must fail replay")
21132        else {
21133            panic!("expected typed replay failure");
21134        };
21135        assert_eq!(failure.reason, "recorded_commands_unconsumed");
21136        assert_eq!(failure.sequence, Some(1));
21137    }
21138
21139    #[test]
21140    fn workflow_context_emits_explicit_child_workflow_contract() {
21141        let ctx = WorkflowContext {
21142            state: Arc::new(Mutex::new(
21143                WorkflowState::new_with_identity(
21144                    Vec::new(),
21145                    Some("wf-parent".to_string()),
21146                    Some("run-parent".to_string()),
21147                    "parent-workers".to_string(),
21148                    DEFAULT_CODEC.to_string(),
21149                    None,
21150                )
21151                .expect("workflow state"),
21152            )),
21153        };
21154        let options = ChildWorkflowOptions::new("python-workers")
21155            .parent_close_policy(ParentClosePolicy::RequestCancel)
21156            .retry_policy(ChildWorkflowRetryPolicy {
21157                max_attempts: Some(3),
21158                backoff_seconds: vec![1, 5],
21159                non_retryable_error_types: vec!["ValidationError".to_string()],
21160            })
21161            .execution_timeout_seconds(600)
21162            .run_timeout_seconds(120);
21163        let mut call = Box::pin(ctx.start_child_workflow(
21164            "python.fulfil-order",
21165            options,
21166            json!([{"order_id": "order-42"}]),
21167        ));
21168        let mut task_context = TaskContext::from_waker(noop_waker_ref());
21169
21170        assert!(matches!(
21171            call.as_mut().poll(&mut task_context),
21172            Poll::Pending
21173        ));
21174        let commands = ctx.take_commands().expect("commands");
21175        assert_eq!(commands.len(), 1);
21176        let command = &commands[0];
21177        assert_eq!(command["type"], "start_child_workflow");
21178        assert_eq!(command["workflow_type"], "python.fulfil-order");
21179        assert_eq!(command["queue"], "python-workers");
21180        assert_eq!(command["parent_close_policy"], "request_cancel");
21181        assert_eq!(command["retry_policy"]["max_attempts"], 3);
21182        assert_eq!(command["execution_timeout_seconds"], 600);
21183        assert_eq!(command["run_timeout_seconds"], 120);
21184        assert_eq!(
21185            decode_wire_value(&command["arguments"], DEFAULT_CODEC).expect("child args"),
21186            json!([{"order_id": "order-42"}])
21187        );
21188    }
21189
21190    fn child_parent_worker() -> Worker {
21191        let client = Client::new("http://127.0.0.1:8080").expect("client");
21192        let mut worker = Worker::new(client, "rust-parent-workers");
21193        worker.register_workflow("rust.parent", |ctx, _input| async move {
21194            let child = ctx
21195                .start_child_workflow(
21196                    "python.child",
21197                    ChildWorkflowOptions::new("python-child-workers")
21198                        .parent_close_policy(ParentClosePolicy::Terminate),
21199                    json!([{"codec_probe": [1, true, "rust"]}]),
21200                )
21201                .await?;
21202            Ok(json!({
21203                "parent_workflow_id": child.parent.workflow_id,
21204                "parent_run_id": child.parent.run_id,
21205                "child_workflow_id": child.child.workflow_id,
21206                "child_run_id": child.child.run_id,
21207                "child_workflow_type": child.child_workflow_type,
21208                "result": child.result,
21209            }))
21210        });
21211        worker
21212    }
21213
21214    fn child_parent_task(event_type: &str, payload: Value) -> WorkflowTask {
21215        WorkflowTask {
21216            task_id: "wft-child-parent".to_string(),
21217            workflow_command_id: None,
21218            workflow_id: Some("wf-parent".to_string()),
21219            run_id: Some("run-parent".to_string()),
21220            workflow_type: "rust.parent".to_string(),
21221            cancel_requested: false,
21222            payload_codec: DEFAULT_CODEC.to_string(),
21223            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21224            history_events: vec![
21225                HistoryEvent {
21226                    event_type: "ChildWorkflowScheduled".to_string(),
21227                    payload: json!({
21228                        "sequence": 1,
21229                        "child_call_id": "call-child",
21230                        "child_workflow_instance_id": "wf-child",
21231                        "child_workflow_run_id": "run-child",
21232                        "child_workflow_type": "python.child",
21233                    }),
21234                    raw: HashMap::new(),
21235                },
21236                HistoryEvent {
21237                    event_type: event_type.to_string(),
21238                    payload,
21239                    raw: HashMap::new(),
21240                },
21241            ],
21242            total_history_events: Some(2),
21243            history_size_bytes: None,
21244            continue_as_new_recommended: None,
21245            history_budget_pressure: None,
21246            next_history_page_token: None,
21247            workflow_task_attempt: 1,
21248            workflow_signal_id: None,
21249            signal_name: None,
21250            signal_arguments: None,
21251            workflow_update_id: None,
21252            update_name: None,
21253            lease_owner: Some("rust-worker".to_string()),
21254        }
21255    }
21256
21257    #[test]
21258    fn committed_child_result_replays_without_starting_a_duplicate() {
21259        let worker = child_parent_worker();
21260        let task = child_parent_task(
21261            "ChildRunCompleted",
21262            json!({
21263                "sequence": 1,
21264                "child_call_id": "call-child",
21265                "child_workflow_instance_id": "wf-child",
21266                "child_workflow_run_id": "run-child",
21267                "child_workflow_type": "python.child",
21268                "payload_codec": DEFAULT_CODEC,
21269                "result": fixture_envelope(json!({"from":"python","ok":true})),
21270            }),
21271        );
21272
21273        for _restart in 0..2 {
21274            let commands = worker
21275                .execute_workflow_task(task.clone())
21276                .expect("replayed parent task");
21277            assert_eq!(commands.len(), 1);
21278            assert_eq!(commands[0]["type"], "complete_workflow");
21279            assert!(!commands
21280                .iter()
21281                .any(|command| command["type"] == "start_child_workflow"));
21282            let output =
21283                decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21284            assert_eq!(output["parent_workflow_id"], "wf-parent");
21285            assert_eq!(output["parent_run_id"], "run-parent");
21286            assert_eq!(output["child_workflow_id"], "wf-child");
21287            assert_eq!(output["child_run_id"], "run-child");
21288            assert_eq!(output["result"], json!({"from": "python", "ok": true}));
21289        }
21290    }
21291
21292    #[test]
21293    fn typed_child_arguments_and_results_survive_replay() {
21294        let client = Client::new("http://127.0.0.1:8080").expect("client");
21295        let mut worker = Worker::new(client, "rust-parent-workers");
21296        worker.register_workflow_avro_value("rust.typed-parent", |ctx, _input| async move {
21297            let child = ctx
21298                .start_child_workflow_avro_value(
21299                    "python.typed-child",
21300                    ChildWorkflowOptions::new("python-workers"),
21301                    AvroValue::Array(vec![typed_fidelity_probe()]),
21302                )
21303                .await?;
21304            Ok(child.result)
21305        });
21306
21307        let initial = worker
21308            .execute_workflow_task(workflow_task(
21309                "rust.typed-parent",
21310                Vec::new(),
21311                DEFAULT_CODEC,
21312            ))
21313            .expect("typed child start");
21314        assert_eq!(initial[0]["type"], "start_child_workflow");
21315        assert_eq!(
21316            decode_wire_avro_value(&initial[0]["arguments"], DEFAULT_CODEC)
21317                .expect("typed child arguments"),
21318            AvroValue::Array(vec![typed_fidelity_probe()])
21319        );
21320
21321        let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
21322            .expect("typed child result");
21323        let task = workflow_task(
21324            "rust.typed-parent",
21325            vec![
21326                history_event(
21327                    "ChildWorkflowScheduled",
21328                    json!({
21329                        "sequence": 1,
21330                        "child_call_id": "call-typed",
21331                        "child_workflow_instance_id": "wf-child",
21332                        "child_workflow_run_id": "run-child",
21333                        "child_workflow_type": "python.typed-child",
21334                    }),
21335                ),
21336                history_event(
21337                    "ChildRunCompleted",
21338                    json!({
21339                        "sequence": 1,
21340                        "child_call_id": "call-typed",
21341                        "child_workflow_instance_id": "wf-child",
21342                        "child_workflow_run_id": "run-child",
21343                        "child_workflow_type": "python.typed-child",
21344                        "payload_codec": DEFAULT_CODEC,
21345                        "result": result,
21346                    }),
21347                ),
21348            ],
21349            DEFAULT_CODEC,
21350        );
21351
21352        let commands = worker
21353            .execute_workflow_task(task)
21354            .expect("typed child replay");
21355        assert_eq!(commands[0]["type"], "complete_workflow");
21356        assert_eq!(
21357            decode_wire_avro_value(&commands[0]["result"], DEFAULT_CODEC)
21358                .expect("typed parent result"),
21359            typed_fidelity_probe()
21360        );
21361    }
21362
21363    #[test]
21364    fn pending_child_replays_after_restart_without_starting_a_duplicate() {
21365        let worker = child_parent_worker();
21366        let mut task = child_parent_task("unused", Value::Null);
21367        task.history_events.truncate(1);
21368        task.total_history_events = Some(1);
21369
21370        for _redelivery_or_restart in 0..2 {
21371            let commands = worker
21372                .execute_workflow_task(task.clone())
21373                .expect("recorded child remains pending");
21374            assert!(
21375                commands.is_empty(),
21376                "recorded pending child must not be started again"
21377            );
21378        }
21379    }
21380
21381    #[test]
21382    fn child_cancellation_becomes_stable_parent_failure_command() {
21383        let worker = child_parent_worker();
21384        let task = child_parent_task(
21385            "ChildRunCancelled",
21386            json!({
21387                "sequence": 1,
21388                "child_workflow_instance_id": "wf-child",
21389                "child_workflow_run_id": "run-child",
21390                "child_workflow_type": "python.child",
21391                "failure_id": "failure-child",
21392                "failure_category": "cancelled",
21393                "message": "cancelled by parent-close policy",
21394            }),
21395        );
21396
21397        let commands = worker
21398            .execute_workflow_task(task)
21399            .expect("parent settlement");
21400        assert_eq!(commands.len(), 1);
21401        assert_eq!(commands[0]["type"], "fail_workflow");
21402        assert_eq!(commands[0]["exception_type"], "ChildWorkflowCancelled");
21403        assert_eq!(
21404            commands[0]["exception"]["properties"]["reason"],
21405            "cancelled"
21406        );
21407        assert_eq!(
21408            commands[0]["exception"]["properties"]["child_workflow_run_id"],
21409            "run-child"
21410        );
21411    }
21412
21413    #[test]
21414    fn workflow_can_handle_typed_child_failure() {
21415        let client = Client::new("http://127.0.0.1:8080").expect("client");
21416        let mut worker = Worker::new(client, "rust-parent-workers");
21417        worker.register_workflow("rust.handled-parent", |ctx, _input| async move {
21418            match ctx
21419                .start_child_workflow(
21420                    "python.child",
21421                    ChildWorkflowOptions::new("python-child-workers"),
21422                    json!([]),
21423                )
21424                .await
21425            {
21426                Err(Error::ChildWorkflowFailed(failure)) => Ok(json!({
21427                    "reason": failure.reason,
21428                    "failure_id": failure.failure_id,
21429                    "exception_class": failure.exception_class,
21430                    "child_run_id": failure.child_workflow_run_id,
21431                })),
21432                Err(error) => Err(error),
21433                Ok(_) => Err(Error::WorkerLoop(
21434                    "child unexpectedly succeeded".to_string(),
21435                )),
21436            }
21437        });
21438        let mut task = child_parent_task(
21439            "ChildRunFailed",
21440            json!({
21441                "sequence": 1,
21442                "child_workflow_instance_id": "wf-child",
21443                "child_workflow_run_id": "run-child",
21444                "child_workflow_type": "python.child",
21445                "failure_id": "failure-child",
21446                "failure_category": "child_workflow",
21447                "message": "payment rejected",
21448                "exception": {
21449                    "type": "PaymentRejected",
21450                    "class": "payments.PaymentRejected",
21451                    "message": "payment rejected"
21452                }
21453            }),
21454        );
21455        task.workflow_type = "rust.handled-parent".to_string();
21456
21457        let commands = worker.execute_workflow_task(task).expect("handled failure");
21458        assert_eq!(commands[0]["type"], "complete_workflow");
21459        let output =
21460            decode_wire_value(&commands[0]["result"], DEFAULT_CODEC).expect("parent output");
21461        assert_eq!(output["reason"], "child_workflow");
21462        assert_eq!(output["failure_id"], "failure-child");
21463        assert_eq!(output["exception_class"], "payments.PaymentRejected");
21464        assert_eq!(output["child_run_id"], "run-child");
21465    }
21466
21467    #[test]
21468    fn rust_hello_world_uses_signal_arguments_from_resume_payload() {
21469        let client = Client::new("http://127.0.0.1:8080").expect("client");
21470        let mut worker = Worker::new(client, "rust-workers");
21471
21472        worker.register_workflow("rust.hello_workflow", |ctx, _input| async move {
21473            let signal = ctx.wait_signal("start").await?;
21474            let name = signal
21475                .first()
21476                .and_then(|value| value.as_str())
21477                .unwrap_or("world");
21478            let greeting = ctx.activity("rust.hello_activity", json!([name])).await?;
21479            Ok(json!({
21480                "greeting": greeting,
21481                "language": "rust"
21482            }))
21483        });
21484
21485        let signal_arguments =
21486            encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC).expect("signal arguments");
21487        let task = WorkflowTask {
21488            task_id: "wft-rust-signal-1".to_string(),
21489            workflow_command_id: None,
21490            workflow_id: Some("wf-rust-hello".to_string()),
21491            run_id: Some("run-rust-hello".to_string()),
21492            workflow_type: "rust.hello_workflow".to_string(),
21493            cancel_requested: false,
21494            payload_codec: DEFAULT_CODEC.to_string(),
21495            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21496            history_events: vec![HistoryEvent {
21497                event_type: "SignalReceived".to_string(),
21498                payload: json!({
21499                    "signal_id": "sig-rust-1",
21500                    "signal_name": "start"
21501                }),
21502                raw: HashMap::new(),
21503            }],
21504            total_history_events: Some(1),
21505            history_size_bytes: None,
21506            continue_as_new_recommended: None,
21507            history_budget_pressure: None,
21508            next_history_page_token: None,
21509            workflow_task_attempt: 1,
21510            workflow_signal_id: Some("sig-rust-1".to_string()),
21511            signal_name: Some("start".to_string()),
21512            signal_arguments: Some(signal_arguments),
21513            workflow_update_id: None,
21514            update_name: None,
21515            lease_owner: Some("rust-worker".to_string()),
21516        };
21517
21518        let commands = worker.execute_workflow_task(task).expect("workflow task");
21519
21520        assert_eq!(commands.len(), 1);
21521        assert_eq!(commands[0]["type"], "schedule_activity");
21522        assert_eq!(commands[0]["activity_type"], "rust.hello_activity");
21523        assert_eq!(
21524            decode_wire_value(&commands[0]["arguments"], DEFAULT_CODEC).expect("activity args"),
21525            json!(["Rust"])
21526        );
21527    }
21528
21529    #[test]
21530    fn workflow_task_appends_paginated_history_events() {
21531        let mut task = WorkflowTask {
21532            task_id: "wft-rust-pages-1".to_string(),
21533            workflow_command_id: None,
21534            workflow_id: Some("wf-rust-pages".to_string()),
21535            run_id: Some("run-rust-pages".to_string()),
21536            workflow_type: "rust.hello_workflow".to_string(),
21537            cancel_requested: false,
21538            payload_codec: DEFAULT_CODEC.to_string(),
21539            arguments: Some(encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("input")),
21540            history_events: vec![HistoryEvent {
21541                event_type: "WorkflowStarted".to_string(),
21542                payload: json!({}),
21543                raw: HashMap::new(),
21544            }],
21545            total_history_events: Some(3),
21546            history_size_bytes: None,
21547            continue_as_new_recommended: None,
21548            history_budget_pressure: None,
21549            next_history_page_token: Some("MQ==".to_string()),
21550            workflow_task_attempt: 1,
21551            workflow_signal_id: None,
21552            signal_name: None,
21553            signal_arguments: None,
21554            workflow_update_id: None,
21555            update_name: None,
21556            lease_owner: Some("rust-worker".to_string()),
21557        };
21558
21559        task.append_history_page(WorkflowTaskHistoryPage {
21560            history_events: vec![
21561                HistoryEvent {
21562                    event_type: "SignalReceived".to_string(),
21563                    payload: json!({
21564                        "signal_id": "sig-rust-1",
21565                        "signal_name": "start",
21566                        "arguments": encode_value_envelope(&json!(["Rust"]), DEFAULT_CODEC)
21567                            .expect("signal arguments")
21568                    }),
21569                    raw: HashMap::new(),
21570                },
21571                HistoryEvent {
21572                    event_type: "MarkerRecorded".to_string(),
21573                    payload: json!({"sequence": 3}),
21574                    raw: HashMap::new(),
21575                },
21576            ],
21577            total_history_events: Some(3),
21578            next_history_page_token: None,
21579        });
21580
21581        assert_eq!(task.history_events.len(), 3);
21582        assert_eq!(task.total_history_events, Some(3));
21583        assert_eq!(task.next_history_page_token, None);
21584
21585        let signal = task
21586            .history_events
21587            .iter()
21588            .find(|event| event.event_type == "SignalReceived")
21589            .expect("signal event");
21590        assert_eq!(
21591            decode_signal_event_arguments(signal, DEFAULT_CODEC).expect("signal arguments"),
21592            vec![AvroValue::String("Rust".to_string())]
21593        );
21594    }
21595
21596    #[tokio::test]
21597    async fn query_handler_reads_ordered_cross_codec_signals_without_commands() {
21598        let client = Client::new("http://127.0.0.1:8080").expect("client");
21599        let mut worker = Worker::new(client, "rust-workers");
21600        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21601        worker.register_query("counter", "current", |ctx, _args| async move {
21602            let mut count = 0_i64;
21603            for signal in ctx.signal_events() {
21604                let value = signal
21605                    .arguments
21606                    .first()
21607                    .and_then(Value::as_i64)
21608                    .unwrap_or_default();
21609                match signal.name.as_str() {
21610                    "increment" => count += value,
21611                    "set" => count = value,
21612                    _ => {}
21613                }
21614            }
21615            Ok(json!(count))
21616        });
21617
21618        let task = QueryTask {
21619            query_task_id: "query-rust-counter".to_string(),
21620            query_task_attempt: 1,
21621            lease_owner: Some("rust-worker".to_string()),
21622            workflow_id: Some("counter-1".to_string()),
21623            run_id: Some("run-counter-1".to_string()),
21624            workflow_type: "counter".to_string(),
21625            query_name: "current".to_string(),
21626            payload_codec: DEFAULT_CODEC.to_string(),
21627            workflow_arguments: Some(
21628                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("workflow input"),
21629            ),
21630            query_arguments: Some(
21631                encode_value_envelope(&json!([]), DEFAULT_CODEC).expect("query arguments"),
21632            ),
21633            history_events: vec![
21634                HistoryEvent {
21635                    event_type: "SignalReceived".to_string(),
21636                    payload: json!({
21637                        "signal_id": "php-signal-1",
21638                        "signal_name": "increment",
21639                        "workflow_sequence": 1,
21640                        "payload_codec": DEFAULT_CODEC,
21641                        "arguments": encode_value_envelope(&json!([3]), DEFAULT_CODEC).expect("php avro signal")
21642                    }),
21643                    raw: HashMap::new(),
21644                },
21645                HistoryEvent {
21646                    event_type: "SignalReceived".to_string(),
21647                    payload: json!({
21648                        "signal_id": "python-signal-2",
21649                        "signal_name": "increment",
21650                        "workflow_sequence": 2,
21651                        "payload_codec": DEFAULT_CODEC,
21652                        "arguments": encode_value_envelope(&json!([5]), DEFAULT_CODEC).expect("python avro signal")
21653                    }),
21654                    raw: HashMap::new(),
21655                },
21656                HistoryEvent {
21657                    event_type: "SignalReceived".to_string(),
21658                    payload: json!({
21659                        "signal_id": "rust-signal-3",
21660                        "signal_name": "set",
21661                        "workflow_sequence": 3,
21662                        "payload_codec": DEFAULT_CODEC,
21663                        "arguments": encode_value_envelope(&json!([0]), DEFAULT_CODEC).expect("rust avro signal")
21664                    }),
21665                    raw: HashMap::new(),
21666                },
21667            ],
21668            history_export: None,
21669            run_status: Some("completed".to_string()),
21670        };
21671
21672        let result = worker.execute_query_task(task).await.expect("query result");
21673        assert_eq!(result.into_json().expect("query projection"), json!(0));
21674    }
21675
21676    #[tokio::test]
21677    async fn replayed_queries_read_running_completed_and_cold_restarted_instance_state() {
21678        let worker = replay_counter_worker();
21679        let running_history = json!([
21680            {
21681                "type": "ActivityCompleted",
21682                "payload": {
21683                    "sequence": 1,
21684                    "activity_type": "load-counter",
21685                    "payload_codec": DEFAULT_CODEC,
21686                    "result": fixture_envelope(json!("loaded"))
21687                }
21688            },
21689            {
21690                "type": "SignalWaitOpened",
21691                "payload": {
21692                    "sequence": 3,
21693                    "signal_name": "increment"
21694                }
21695            },
21696            {
21697                "type": "SignalReceived",
21698                "payload": {
21699                    "signal_id": "signal-3",
21700                    "signal_name": "increment",
21701                    "workflow_sequence": 2,
21702                    "payload_codec": DEFAULT_CODEC,
21703                    "arguments": fixture_envelope(json!([3]))
21704                }
21705            },
21706            {
21707                "type": "SignalApplied",
21708                "payload": {
21709                    "sequence": 3,
21710                    "signal_id": "signal-3",
21711                    "signal_name": "increment",
21712                    "payload_codec": DEFAULT_CODEC,
21713                    "value": fixture_envelope(json!([3]))
21714                }
21715            }
21716        ]);
21717
21718        let running = worker
21719            .execute_query_task(replay_counter_query(
21720                "current",
21721                running_history.clone(),
21722                "running",
21723            ))
21724            .await
21725            .expect("running replay query");
21726        assert_eq!(
21727            running.clone().into_json().expect("query projection"),
21728            json!({"loaded": "loaded", "count": 3, "finished": false})
21729        );
21730
21731        let detached = worker
21732            .execute_query_task(replay_counter_query(
21733                "detached-mutation",
21734                running_history.clone(),
21735                "running",
21736            ))
21737            .await
21738            .expect("query mutates only its detached state clone");
21739        assert_eq!(detached.into_json().expect("query projection"), json!(999));
21740        let failed = worker
21741            .execute_query_task(replay_counter_query(
21742                "failed-mutation",
21743                running_history.clone(),
21744                "running",
21745            ))
21746            .await
21747            .expect_err("failed query");
21748        assert_eq!(failed.reason, "query_rejected");
21749        let unchanged = worker
21750            .execute_query_task(replay_counter_query("current", running_history, "running"))
21751            .await
21752            .expect("later query reconstructs unchanged state");
21753        assert_eq!(unchanged, running);
21754
21755        let restarted_worker = replay_counter_worker();
21756        let empty_arguments = fixture_envelope(json!([]));
21757        let loaded_result = fixture_envelope(json!("loaded"));
21758        let signal_three = fixture_blob(json!([3]));
21759        let signal_five = fixture_blob(json!([5]));
21760        let restarted_task: QueryTask = serde_json::from_value(json!({
21761            "query_task_id": "query-after-restart",
21762            "workflow_id": "counter-1",
21763            "run_id": "run-counter-1",
21764            "workflow_type": "replay-counter",
21765            "query_name": "current",
21766            "payload_codec": DEFAULT_CODEC,
21767            "workflow_arguments": empty_arguments.clone(),
21768            "query_arguments": empty_arguments,
21769            "history_events": [],
21770            "history_export": {
21771                "payloads": {"codec": DEFAULT_CODEC},
21772                "history_events": [
21773                    {
21774                        "type": "ActivityCompleted",
21775                        "payload": {
21776                            "sequence": 1,
21777                            "activity_type": "load-counter",
21778                            "payload_codec": DEFAULT_CODEC,
21779                            "result": null
21780                        }
21781                    },
21782                    {
21783                        "type": "SignalWaitOpened",
21784                        "payload": {
21785                            "sequence": 3,
21786                            "signal_name": "increment"
21787                        }
21788                    },
21789                    {
21790                        "type": "SignalReceived",
21791                        "payload": {
21792                            "signal_id": "signal-3",
21793                            "signal_name": "increment",
21794                            "workflow_sequence": 2
21795                        }
21796                    },
21797                    {
21798                        "type": "SignalApplied",
21799                        "payload": {
21800                            "sequence": 3,
21801                            "signal_id": "signal-3",
21802                            "signal_name": "increment"
21803                        }
21804                    },
21805                    {
21806                        "type": "SignalWaitOpened",
21807                        "payload": {
21808                            "sequence": 5,
21809                            "signal_name": "increment"
21810                        }
21811                    },
21812                    {
21813                        "type": "SignalReceived",
21814                        "payload": {
21815                            "signal_id": "signal-5",
21816                            "signal_name": "increment",
21817                            "workflow_sequence": 4
21818                        }
21819                    },
21820                    {
21821                        "type": "SignalApplied",
21822                        "payload": {
21823                            "sequence": 5,
21824                            "signal_id": "signal-5",
21825                            "signal_name": "increment"
21826                        }
21827                    }
21828                ],
21829                "activities": [{
21830                    "sequence": 1,
21831                    "activity_type": "load-counter",
21832                    "payload_codec": DEFAULT_CODEC,
21833                    "result": loaded_result
21834                }],
21835                "signals": [
21836                    {
21837                        "id": "signal-3",
21838                        "name": "increment",
21839                        "workflow_sequence": 2,
21840                        "payload_codec": DEFAULT_CODEC,
21841                        "arguments": signal_three
21842                    },
21843                    {
21844                        "id": "signal-5",
21845                        "name": "increment",
21846                        "workflow_sequence": 4,
21847                        "payload_codec": DEFAULT_CODEC,
21848                        "arguments": signal_five
21849                    }
21850                ]
21851            },
21852            "run_status": "completed"
21853        }))
21854        .expect("cold replay query task");
21855        let completed = restarted_worker
21856            .execute_query_task(restarted_task)
21857            .await
21858            .expect("completed cold replay query");
21859        assert_eq!(
21860            completed.into_json().expect("query projection"),
21861            json!({"loaded": "loaded", "count": 8, "finished": true})
21862        );
21863    }
21864
21865    #[tokio::test]
21866    async fn replayed_query_replay_failures_are_machine_readable() {
21867        let worker = replay_counter_worker();
21868        let task = replay_counter_query(
21869            "current",
21870            json!([{
21871                "type": "ActivityCompleted",
21872                "payload": {
21873                    "sequence": 1,
21874                    "payload_codec": DEFAULT_CODEC,
21875                    "result": {"codec": DEFAULT_CODEC, "blob": "{"}
21876                }
21877            }]),
21878            "running",
21879        );
21880        let failure = worker
21881            .execute_query_task(task)
21882            .await
21883            .expect_err("invalid replay history payload");
21884        assert_eq!(failure.reason, "query_payload_decode_failed");
21885        assert_eq!(failure.failure_type, "QueryPayloadDecodeFailed");
21886        assert!(failure.message.contains("invalid_payload_framing"));
21887    }
21888
21889    #[tokio::test]
21890    async fn query_task_restores_compact_history_from_export() {
21891        let client = Client::new("http://127.0.0.1:8080").expect("client");
21892        let mut worker = Worker::new(client, "rust-workers");
21893        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21894        worker.register_query("counter", "current", |ctx, _args| async move {
21895            Ok(json!(ctx.signals("increment")[0][0]))
21896        });
21897        let empty_arguments = fixture_envelope(json!([]));
21898        let exported_signal = fixture_blob(json!([9]));
21899        let task: QueryTask = serde_json::from_value(json!({
21900            "query_task_id": "query-export",
21901            "workflow_type": "counter",
21902            "query_name": "current",
21903            "payload_codec": DEFAULT_CODEC,
21904            "workflow_arguments": empty_arguments.clone(),
21905            "query_arguments": empty_arguments,
21906            "history_events": [],
21907            "history_export": {
21908                "payloads": {"codec": DEFAULT_CODEC},
21909                "history_events": [{
21910                    "type": "SignalReceived",
21911                    "payload": {"signal_id": "signal-export", "signal_name": "increment"}
21912                }],
21913                "signals": [{
21914                    "id": "signal-export",
21915                    "name": "increment",
21916                    "status": "applied",
21917                    "workflow_sequence": 1,
21918                    "payload_codec": DEFAULT_CODEC,
21919                    "arguments": exported_signal
21920                }]
21921            }
21922        }))
21923        .expect("query task");
21924
21925        let result = worker.execute_query_task(task).await.expect("query result");
21926        assert_eq!(result.into_json().expect("query projection"), json!(9));
21927    }
21928
21929    #[tokio::test]
21930    async fn query_task_failures_have_stable_reasons() {
21931        let client = Client::new("http://127.0.0.1:8080").expect("client");
21932        let mut worker = Worker::new(client, "rust-workers");
21933        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21934        worker.register_query(
21935            "counter",
21936            "current",
21937            |_ctx, _args| async move { Ok(json!(0)) },
21938        );
21939
21940        let base_task = QueryTask {
21941            query_task_id: "query-errors".to_string(),
21942            query_task_attempt: 1,
21943            lease_owner: None,
21944            workflow_id: Some("counter-errors".to_string()),
21945            run_id: Some("run-errors".to_string()),
21946            workflow_type: "counter".to_string(),
21947            query_name: "missing".to_string(),
21948            payload_codec: DEFAULT_CODEC.to_string(),
21949            workflow_arguments: Some(fixture_envelope(json!([]))),
21950            query_arguments: Some(fixture_envelope(json!([]))),
21951            history_events: Vec::new(),
21952            history_export: None,
21953            run_status: Some("running".to_string()),
21954        };
21955
21956        let unknown = worker
21957            .execute_query_task(base_task.clone())
21958            .await
21959            .expect_err("unknown query");
21960        assert_eq!(unknown.reason, "rejected_unknown_query");
21961
21962        let mut malformed = base_task;
21963        malformed.query_name = "current".to_string();
21964        malformed.query_arguments = Some(json!({"codec": DEFAULT_CODEC, "blob": "{"}));
21965        let malformed = worker
21966            .execute_query_task(malformed)
21967            .await
21968            .expect_err("malformed payload");
21969        assert_eq!(malformed.reason, "query_payload_decode_failed");
21970
21971        let client = Client::new("http://127.0.0.1:8080").expect("client");
21972        let mut unavailable_worker = Worker::new(client, "rust-workers");
21973        unavailable_worker
21974            .register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
21975        let empty_arguments = fixture_envelope(json!([]));
21976        let unavailable_task: QueryTask = serde_json::from_value(json!({
21977            "query_task_id": "query-unavailable",
21978            "workflow_type": "counter",
21979            "query_name": "current",
21980            "payload_codec": DEFAULT_CODEC,
21981            "workflow_arguments": empty_arguments.clone(),
21982            "query_arguments": empty_arguments
21983        }))
21984        .expect("query task");
21985        let unavailable = unavailable_worker
21986            .execute_query_task(unavailable_task)
21987            .await
21988            .expect_err("query handler unavailable");
21989        assert_eq!(unavailable.reason, "query_handler_unavailable");
21990    }
21991
21992    #[tokio::test]
21993    async fn client_query_decodes_result_and_typed_failure() {
21994        let server = MockWorkerServer::start();
21995        let client = Client::builder(server.base_url())
21996            .timeout(Duration::from_secs(2))
21997            .build()
21998            .expect("client");
21999
22000        let result = client
22001            .query_workflow("counter-1", "current", json!([]))
22002            .await
22003            .expect("query result");
22004        assert_eq!(result, json!({"count": 8}));
22005
22006        let error = client
22007            .query_workflow("counter-1", "missing", json!([]))
22008            .await
22009            .expect_err("unknown query");
22010        let Error::QueryFailed(failure) = error else {
22011            panic!("expected typed query failure");
22012        };
22013        assert_eq!(failure.status, 404);
22014        assert_eq!(failure.reason, "rejected_unknown_query");
22015    }
22016
22017    #[tokio::test]
22018    async fn public_client_surfaces_send_and_receive_lossless_avro_values() {
22019        let server = MockWorkerServer::start();
22020        let client = Client::builder(server.base_url())
22021            .timeout(Duration::from_secs(2))
22022            .build()
22023            .expect("client");
22024        let arguments = AvroValue::Array(vec![typed_fidelity_probe()]);
22025
22026        client
22027            .start_workflow(
22028                "typed.echo",
22029                "rust-workers",
22030                "typed-start",
22031                arguments.clone(),
22032            )
22033            .await
22034            .expect("typed workflow start");
22035        assert_eq!(
22036            decode_wire_avro_value(
22037                &server.request_body("/api/workflows")["input"],
22038                DEFAULT_CODEC,
22039            )
22040            .expect("typed start input"),
22041            arguments
22042        );
22043
22044        client
22045            .signal_workflow("typed-1", "changed", arguments.clone())
22046            .await
22047            .expect("typed signal");
22048        assert_eq!(
22049            decode_wire_avro_value(
22050                &server.request_body("/api/workflows/typed-1/signal/changed")["input"],
22051                DEFAULT_CODEC,
22052            )
22053            .expect("typed signal input"),
22054            arguments
22055        );
22056
22057        assert_eq!(
22058            client
22059                .query_workflow_avro_value("typed-1", "inspect", arguments.clone())
22060                .await
22061                .expect("typed query"),
22062            typed_fidelity_probe()
22063        );
22064        assert_eq!(
22065            decode_wire_avro_value(
22066                &server.request_body("/api/workflows/typed-1/query/inspect")["input"],
22067                DEFAULT_CODEC,
22068            )
22069            .expect("typed query input"),
22070            arguments
22071        );
22072
22073        assert_eq!(
22074            client
22075                .update_workflow_avro_value(
22076                    "typed-1",
22077                    "replace",
22078                    arguments.clone(),
22079                    Some("typed-request"),
22080                )
22081                .await
22082                .expect("typed update"),
22083            typed_fidelity_probe()
22084        );
22085        let update = server.request_body("/api/workflows/typed-1/update/replace");
22086        assert_eq!(update["request_id"], "typed-request");
22087        assert_eq!(
22088            decode_wire_avro_value(&update["input"], DEFAULT_CODEC).expect("typed update input"),
22089            arguments
22090        );
22091
22092        let handle = WorkflowHandle {
22093            client: client.clone(),
22094            workflow_id: "typed-1".to_string(),
22095            run_id: Some("run-typed-1".to_string()),
22096            workflow_type: "typed.echo".to_string(),
22097        };
22098        assert_eq!(
22099            handle
22100                .result_avro_value(WorkflowResultOptions::default())
22101                .await
22102                .expect("typed workflow result"),
22103            typed_fidelity_probe()
22104        );
22105
22106        client
22107            .complete_activity_task(
22108                "activity-typed",
22109                "attempt-typed",
22110                "rust-worker",
22111                typed_fidelity_probe(),
22112                DEFAULT_CODEC,
22113            )
22114            .await
22115            .expect("typed activity completion");
22116        assert_eq!(
22117            decode_wire_avro_value(
22118                &server.request_body("/api/worker/activity-tasks/activity-typed/complete")
22119                    ["result"],
22120                DEFAULT_CODEC,
22121            )
22122            .expect("typed activity result"),
22123            typed_fidelity_probe()
22124        );
22125        client
22126            .fail_activity_task(
22127                "activity-typed",
22128                "attempt-typed",
22129                "rust-worker",
22130                "typed failure",
22131                true,
22132            )
22133            .await
22134            .expect("activity failure");
22135    }
22136
22137    #[tokio::test]
22138    async fn lifecycle_commands_support_instance_and_selected_run_targets() {
22139        let server = MockWorkerServer::start();
22140        let client = Client::builder(server.base_url())
22141            .timeout(Duration::from_secs(2))
22142            .build()
22143            .expect("client");
22144
22145        let options = WorkflowCommandOptions::new()
22146            .reason("cleanup requested")
22147            .request_id("cancel-17");
22148        let cancelled = client
22149            .cancel_workflow("wf-lifecycle", options)
22150            .await
22151            .expect("instance cancellation");
22152        assert_eq!(cancelled.command, WorkflowCommandKind::Cancel);
22153        assert_eq!(cancelled.run_id.as_deref(), Some("run-current"));
22154        assert_eq!(cancelled.outcome.as_deref(), Some("cancelled"));
22155        assert_eq!(
22156            server.request_body("/api/workflows/wf-lifecycle/cancel"),
22157            json!({"reason":"cleanup requested","request_id":"cancel-17"})
22158        );
22159
22160        let terminated = client
22161            .terminate_workflow(
22162                "wf-lifecycle",
22163                WorkflowCommandOptions::new().reason("forced stop"),
22164            )
22165            .await
22166            .expect("instance termination");
22167        assert_eq!(terminated.command, WorkflowCommandKind::Terminate);
22168        assert_eq!(terminated.outcome.as_deref(), Some("terminated"));
22169
22170        client
22171            .cancel_workflow_run(
22172                "wf-lifecycle",
22173                "run-current",
22174                WorkflowCommandOptions::default(),
22175            )
22176            .await
22177            .expect("selected run cancellation");
22178        client
22179            .terminate_workflow_run(
22180                "wf-lifecycle",
22181                "run-current",
22182                WorkflowCommandOptions::default(),
22183            )
22184            .await
22185            .expect("selected run termination");
22186
22187        for (command, error) in [
22188            (
22189                WorkflowCommandKind::Cancel,
22190                client
22191                    .cancel_workflow_run(
22192                        "wf-lifecycle",
22193                        "run-stale",
22194                        WorkflowCommandOptions::default(),
22195                    )
22196                    .await
22197                    .expect_err("stale cancellation must be rejected"),
22198            ),
22199            (
22200                WorkflowCommandKind::Terminate,
22201                client
22202                    .terminate_workflow_run(
22203                        "wf-lifecycle",
22204                        "run-stale",
22205                        WorkflowCommandOptions::default(),
22206                    )
22207                    .await
22208                    .expect_err("stale termination must be rejected"),
22209            ),
22210        ] {
22211            let Error::WorkflowCommandRejected(rejection) = error else {
22212                panic!("expected typed command rejection");
22213            };
22214            assert_eq!(rejection.command, command);
22215            assert_eq!(rejection.status, 409);
22216            assert_eq!(rejection.reason, "historical_run_command_rejected");
22217            assert_eq!(rejection.run_id.as_deref(), Some("run-stale"));
22218            assert_eq!(rejection.target_scope.as_deref(), Some("run"));
22219        }
22220    }
22221
22222    #[tokio::test]
22223    async fn workflow_start_options_send_server_enforced_deadlines() {
22224        let server = MockWorkerServer::start();
22225        let client = Client::builder(server.base_url())
22226            .timeout(Duration::from_secs(2))
22227            .build()
22228            .expect("client");
22229
22230        let handle = client
22231            .start_workflow_with_options(
22232                "rust.timeout",
22233                "rust-timeouts",
22234                "wf-start-options",
22235                WorkflowStartOptions::new()
22236                    .execution_timeout_seconds(30)
22237                    .run_timeout_seconds(1),
22238                json!([]),
22239            )
22240            .await
22241            .expect("workflow start");
22242
22243        assert_eq!(handle.run_id.as_deref(), Some("run-start-options"));
22244        let body = server.request_body("/api/workflows");
22245        assert_eq!(body["execution_timeout_seconds"], 30);
22246        assert_eq!(body["run_timeout_seconds"], 1);
22247
22248        let invalid = client
22249            .start_workflow_with_options(
22250                "rust.timeout",
22251                "rust-timeouts",
22252                "wf-invalid-options",
22253                WorkflowStartOptions::new()
22254                    .execution_timeout_seconds(1)
22255                    .run_timeout_seconds(2),
22256                json!([]),
22257            )
22258            .await
22259            .expect_err("invalid deadline ordering");
22260        assert!(invalid
22261            .to_string()
22262            .contains("run_timeout_seconds cannot exceed execution_timeout_seconds"));
22263    }
22264
22265    #[tokio::test]
22266    async fn workflow_result_returns_each_typed_terminal_outcome() {
22267        let server = MockWorkerServer::start();
22268        let client = Client::builder(server.base_url())
22269            .timeout(Duration::from_secs(2))
22270            .build()
22271            .expect("client");
22272        let options = WorkflowResultOptions {
22273            poll_interval: Duration::ZERO,
22274            timeout: Duration::from_secs(1),
22275        };
22276
22277        let failed = WorkflowHandle {
22278            client: client.clone(),
22279            workflow_id: "wf-failed".to_string(),
22280            run_id: Some("run-failed".to_string()),
22281            workflow_type: "failure".to_string(),
22282        }
22283        .result(options)
22284        .await
22285        .expect_err("failed outcome");
22286        let Error::WorkflowFailed(failure) = failed else {
22287            panic!("expected WorkflowFailed");
22288        };
22289        assert_eq!(failure.workflow_id, "wf-failed");
22290        assert_eq!(failure.run_id.as_deref(), Some("run-failed"));
22291        assert_eq!(failure.failure_id.as_deref(), Some("failure-17"));
22292        assert_eq!(failure.failure_category.as_deref(), Some("application"));
22293        assert_eq!(failure.exception_type.as_deref(), Some("PaymentError"));
22294        assert_eq!(
22295            failure.exception_class.as_deref(),
22296            Some("billing::PaymentError")
22297        );
22298        assert_eq!(failure.non_retryable, Some(true));
22299
22300        for (workflow_id, expected_kind, expected_reason) in [
22301            (
22302                "wf-cancelled",
22303                WorkflowTerminalKind::Cancelled,
22304                "cleanup requested",
22305            ),
22306            (
22307                "wf-terminated",
22308                WorkflowTerminalKind::Terminated,
22309                "forced stop",
22310            ),
22311            (
22312                "wf-timed-out",
22313                WorkflowTerminalKind::TimedOut,
22314                "run_timeout",
22315            ),
22316        ] {
22317            let error = WorkflowHandle {
22318                client: client.clone(),
22319                workflow_id: workflow_id.to_string(),
22320                run_id: None,
22321                workflow_type: "terminal".to_string(),
22322            }
22323            .result(options)
22324            .await
22325            .expect_err("typed terminal outcome");
22326            let outcome = match error {
22327                Error::WorkflowCancelled(outcome) => outcome,
22328                Error::WorkflowTerminated(outcome) => outcome,
22329                Error::WorkflowTimedOut(outcome) => outcome,
22330                other => panic!("unexpected terminal error: {other}"),
22331            };
22332            assert_eq!(outcome.kind, expected_kind);
22333            assert_eq!(outcome.workflow_id, workflow_id);
22334            assert_eq!(outcome.reason, expected_reason);
22335        }
22336
22337        let wait_timeout = WorkflowHandle {
22338            client,
22339            workflow_id: "wf-waiting".to_string(),
22340            run_id: Some("run-waiting".to_string()),
22341            workflow_type: "waiting".to_string(),
22342        }
22343        .result(WorkflowResultOptions {
22344            poll_interval: Duration::ZERO,
22345            timeout: Duration::ZERO,
22346        })
22347        .await
22348        .expect_err("client wait timeout");
22349        let Error::WorkflowTimedOut(timeout) = wait_timeout else {
22350            panic!("expected typed client timeout");
22351        };
22352        assert_eq!(timeout.reason, "result_wait_timeout");
22353        assert_eq!(timeout.failure_category.as_deref(), Some("client_timeout"));
22354        assert_eq!(timeout.run_id.as_deref(), Some("run-waiting"));
22355    }
22356
22357    #[tokio::test]
22358    async fn workflow_result_follows_chain_and_selected_result_preserves_history() {
22359        let server = MockWorkerServer::start();
22360        let client = Client::builder(server.base_url())
22361            .timeout(Duration::from_secs(2))
22362            .build()
22363            .expect("client");
22364
22365        let handle = WorkflowHandle {
22366            client,
22367            workflow_id: "wf-selected".to_string(),
22368            run_id: Some("run-selected".to_string()),
22369            workflow_type: "selected".to_string(),
22370        };
22371        let options = WorkflowResultOptions {
22372            poll_interval: Duration::ZERO,
22373            timeout: Duration::from_secs(1),
22374        };
22375
22376        let current = handle
22377            .result(options)
22378            .await
22379            .expect("instance result follows the current run");
22380        assert_eq!(current, json!("current run output"));
22381
22382        let error = handle
22383            .result_selected_run(options)
22384            .await
22385            .expect_err("the selected run is cancelled even though the current run completed");
22386
22387        let Error::WorkflowCancelled(outcome) = error else {
22388            panic!("expected selected run cancellation");
22389        };
22390        assert_eq!(outcome.run_id.as_deref(), Some("run-selected"));
22391        assert_eq!(outcome.reason, "selected run cancelled");
22392        assert_eq!(
22393            server.request_count("/api/workflows/wf-selected/runs/run-selected"),
22394            1
22395        );
22396        assert_eq!(server.request_count("/api/workflows/wf-selected"), 1);
22397    }
22398
22399    #[tokio::test]
22400    async fn poll_responses_decode_http_conflict_drain_as_a_stable_stop() {
22401        let server = MockWorkerServer::draining_polls();
22402        let client = Client::builder(server.base_url())
22403            .timeout(Duration::from_secs(2))
22404            .build()
22405            .expect("client");
22406
22407        let workflow = client
22408            .poll_workflow_task_response("draining-worker", "rust-workers", Duration::ZERO)
22409            .await
22410            .expect("workflow drain response");
22411        let activity = client
22412            .poll_activity_task_response("draining-worker", "rust-workers", Duration::ZERO)
22413            .await
22414            .expect("activity drain response");
22415        let query = client
22416            .poll_query_task_response("draining-worker", "rust-workers", Duration::ZERO)
22417            .await
22418            .expect("query drain response");
22419
22420        for outcome in [workflow.outcome(), activity.outcome(), query.outcome()] {
22421            assert_eq!(
22422                outcome,
22423                WorkerPollOutcome::Stop {
22424                    poll_status: Some("draining".to_string()),
22425                    reason: Some("worker_draining".to_string()),
22426                }
22427            );
22428        }
22429
22430        assert!(client
22431            .poll_workflow_task("draining-worker", "rust-workers", Duration::ZERO)
22432            .await
22433            .expect("compatibility poll")
22434            .is_none());
22435    }
22436
22437    #[tokio::test]
22438    async fn managed_worker_honors_drain_stop_for_every_task_family() {
22439        let server = MockWorkerServer::draining_polls();
22440        let client = Client::builder(server.base_url())
22441            .timeout(Duration::from_secs(2))
22442            .build()
22443            .expect("client");
22444
22445        let mut workflow_worker = Worker::new(client.clone(), "rust-workers")
22446            .worker_id("draining-workflow-worker")
22447            .poll_timeout(Duration::ZERO);
22448        workflow_worker.register_workflow("counter", |_ctx, _args| async { Ok(Value::Null) });
22449        workflow_worker
22450            .run()
22451            .await
22452            .expect("workflow drain is a clean stop");
22453
22454        let mut activity_worker = Worker::new(client.clone(), "rust-workers")
22455            .worker_id("draining-activity-worker")
22456            .poll_timeout(Duration::ZERO);
22457        activity_worker.register_activity("write", |_ctx, _args| async { Ok(Value::Null) });
22458        activity_worker
22459            .run()
22460            .await
22461            .expect("activity drain is a clean stop");
22462
22463        let mut query_worker = Worker::new(client, "rust-workers")
22464            .worker_id("draining-query-worker")
22465            .poll_timeout(Duration::ZERO);
22466        query_worker.register_query("counter", "current", |_ctx, _args| async {
22467            Ok(Value::Null)
22468        });
22469        query_worker
22470            .run()
22471            .await
22472            .expect("query drain is a clean stop");
22473    }
22474
22475    #[tokio::test]
22476    async fn activity_cancellation_and_late_completion_remain_machine_readable() {
22477        let server = MockWorkerServer::start();
22478        let client = Client::builder(server.base_url())
22479            .timeout(Duration::from_secs(2))
22480            .build()
22481            .expect("client");
22482
22483        let heartbeat = client
22484            .heartbeat_activity_task(
22485                "activity-cancel",
22486                "attempt-cancel",
22487                "rust-worker",
22488                typed_fidelity_probe(),
22489            )
22490            .await
22491            .expect("cancellation heartbeat");
22492        assert!(heartbeat.cancel_requested);
22493        assert!(heartbeat.should_stop());
22494        assert_eq!(heartbeat.reason.as_deref(), Some("run_cancelled"));
22495        assert_eq!(heartbeat.run_closed_reason.as_deref(), Some("cancelled"));
22496        let heartbeat_body =
22497            server.request_body("/api/worker/activity-tasks/activity-cancel/heartbeat");
22498        assert_eq!(heartbeat_body["details"]["codec"], DEFAULT_CODEC);
22499        assert_eq!(
22500            decode_wire_avro_value(&heartbeat_body["details"], DEFAULT_CODEC)
22501                .expect("typed heartbeat details"),
22502            typed_fidelity_probe()
22503        );
22504
22505        let error = client
22506            .complete_activity_task(
22507                "activity-cancel",
22508                "attempt-cancel",
22509                "rust-worker",
22510                json!({"late":true}),
22511                DEFAULT_CODEC,
22512            )
22513            .await
22514            .expect_err("late completion must be refused");
22515        assert!(activity_task_rejection_is_final(&error));
22516        let Error::ActivityTaskRejected(rejection) = error else {
22517            panic!("expected typed activity rejection");
22518        };
22519        assert_eq!(rejection.status, 409);
22520        assert_eq!(rejection.reason, "run_cancelled");
22521        assert!(rejection.cancel_requested);
22522        assert_eq!(rejection.can_continue, Some(false));
22523    }
22524
22525    #[tokio::test]
22526    async fn managed_worker_survives_late_completion_and_restart_during_cancellation() {
22527        let server = MockWorkerServer::cancelled_activity();
22528        let client = Client::builder(server.base_url())
22529            .timeout(Duration::from_secs(2))
22530            .build()
22531            .expect("client");
22532        let cancellation_observed = Arc::new(AtomicBool::new(false));
22533        let observed = Arc::clone(&cancellation_observed);
22534        let mut worker = Worker::new(client.clone(), "rust-workers")
22535            .worker_id("rust-cancel-worker")
22536            .poll_timeout(Duration::from_millis(10));
22537        worker.register_activity("cancel-aware", move |ctx, _args| {
22538            let observed = Arc::clone(&observed);
22539            async move {
22540                let heartbeat = ctx.heartbeat(json!({"stage":"running"})).await?;
22541                observed.store(heartbeat.should_stop(), Ordering::SeqCst);
22542                Ok(json!({"late":"completion"}))
22543            }
22544        });
22545
22546        assert_eq!(
22547            worker.run_once().await.expect("cancelled attempt handled"),
22548            1
22549        );
22550        assert!(cancellation_observed.load(Ordering::SeqCst));
22551        assert_eq!(
22552            server.request_count("/api/worker/activity-tasks/activity-cancel/complete"),
22553            1
22554        );
22555
22556        let mut restarted = Worker::new(client, "rust-workers")
22557            .worker_id("rust-cancel-worker-restarted")
22558            .poll_timeout(Duration::from_millis(10));
22559        restarted.register_activity("cancel-aware", |_ctx, _args| async move { Ok(Value::Null) });
22560        assert_eq!(
22561            restarted
22562                .run_once()
22563                .await
22564                .expect("replacement worker continues polling"),
22565            0
22566        );
22567    }
22568
22569    #[tokio::test]
22570    async fn managed_worker_absorbs_selected_run_terminal_timeout_completion_race() {
22571        let response = r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"outcome":"completed","recorded":false,"run_id":"run-selected-timeout","run_status":"failed","created_task_ids":[],"reason":"run_timed_out"}"#;
22572        let server = MockWorkerServer::workflow_completion("409 Conflict", response);
22573        let client = Client::builder(server.base_url())
22574            .timeout(Duration::from_secs(2))
22575            .build()
22576            .expect("client");
22577
22578        let direct_error = client
22579            .complete_workflow_task(
22580                "workflow-timeout-task",
22581                "timeout-worker",
22582                3,
22583                vec![json!({
22584                    "type": "complete_workflow",
22585                    "result": fixture_envelope(Value::Null)
22586                })],
22587            )
22588            .await
22589            .expect_err("the low-level client preserves the completion rejection");
22590        let Error::Http { status, body } = direct_error else {
22591            panic!("expected the original HTTP completion rejection");
22592        };
22593        assert_eq!(status, reqwest::StatusCode::CONFLICT);
22594        assert_eq!(
22595            serde_json::from_str::<Value>(&body).expect("response body")["reason"],
22596            "run_timed_out"
22597        );
22598
22599        let mut worker = Worker::new(client, "rust-workers")
22600            .worker_id("timeout-worker")
22601            .poll_timeout(Duration::from_millis(10));
22602        worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22603            Ok(json!({"late": "result"}))
22604        });
22605
22606        assert_eq!(
22607            worker
22608                .run_once()
22609                .await
22610                .expect("authoritative selected-run timeout settles the tick"),
22611            1
22612        );
22613        assert_eq!(
22614            server.request_count("/api/worker/workflow-tasks/workflow-timeout-task/complete"),
22615            2,
22616            "both the direct client proof and managed worker must see the rejection"
22617        );
22618    }
22619
22620    #[tokio::test]
22621    async fn managed_worker_does_not_swallow_nearby_completion_errors() {
22622        for (name, status, response) in [
22623            ("bare conflict", "409 Conflict", r#"{"message":"conflict"}"#),
22624            (
22625                "command was recorded",
22626                "409 Conflict",
22627                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":true,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22628            ),
22629            (
22630                "lease conflict",
22631                "409 Conflict",
22632                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"lease_expired"}"#,
22633            ),
22634            (
22635                "nonterminal run",
22636                "409 Conflict",
22637                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"waiting","reason":"run_timed_out"}"#,
22638            ),
22639            (
22640                "different selected run",
22641                "409 Conflict",
22642                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-reused-workflow-current","run_status":"failed","reason":"run_timed_out"}"#,
22643            ),
22644            (
22645                "different task attempt",
22646                "409 Conflict",
22647                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":4,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22648            ),
22649            (
22650                "authentication failure",
22651                "401 Unauthorized",
22652                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22653            ),
22654            (
22655                "authorization failure",
22656                "403 Forbidden",
22657                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22658            ),
22659            (
22660                "protocol failure",
22661                "400 Bad Request",
22662                r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.2","requested_version":"1.3"}"#,
22663            ),
22664            (
22665                "malformed command",
22666                "422 Unprocessable Entity",
22667                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22668            ),
22669            (
22670                "transient server failure",
22671                "503 Service Unavailable",
22672                r#"{"task_id":"workflow-timeout-task","workflow_task_attempt":3,"recorded":false,"run_id":"run-selected-timeout","run_status":"failed","reason":"run_timed_out"}"#,
22673            ),
22674        ] {
22675            let server = MockWorkerServer::workflow_completion(status, response);
22676            let client = Client::builder(server.base_url())
22677                .timeout(Duration::from_secs(2))
22678                .build()
22679                .expect("client");
22680            let mut worker = Worker::new(client, "rust-workers")
22681                .worker_id("timeout-worker")
22682                .poll_timeout(Duration::from_millis(10));
22683            worker.register_workflow("timeout.workflow", |_ctx, _input| async move {
22684                Ok(json!({"late": "result"}))
22685            });
22686
22687            let error = worker
22688                .run_once()
22689                .await
22690                .expect_err(&format!("{name} must remain an error"));
22691            assert!(
22692                matches!(error, Error::Http { .. } | Error::Protocol(_)),
22693                "{name} returned an unexpected error variant: {error}"
22694            );
22695        }
22696    }
22697
22698    #[tokio::test]
22699    async fn worker_deregistration_uses_worker_plane_method_path_headers_and_result() {
22700        let server = MockWorkerServer::start();
22701        let client = Client::builder(server.base_url())
22702            .worker_token(Some("worker-secret".to_string()))
22703            .namespace("orders")
22704            .timeout(Duration::from_secs(2))
22705            .build()
22706            .expect("client");
22707        let path = "/api/worker/registrations/worker%2F%CE%B1%20space";
22708
22709        let result = client
22710            .deregister_worker_registration("worker/α space")
22711            .await
22712            .expect("deregister worker registration");
22713
22714        assert_eq!(server.method_for(path).as_deref(), Some("DELETE"));
22715        assert_eq!(
22716            server.worker_protocol_for(path).as_deref(),
22717            Some(WORKER_PROTOCOL_VERSION)
22718        );
22719        assert_eq!(server.control_protocol_for(path), None);
22720        assert_eq!(server.namespace_for(path).as_deref(), Some("orders"));
22721        assert_eq!(
22722            server.authorization_for(path).as_deref(),
22723            Some("Bearer worker-secret")
22724        );
22725        assert_eq!(
22726            result,
22727            WorkerDeregistrationEnvelope {
22728                worker_id: "deregistered-worker".to_string(),
22729                outcome: "deregistered".to_string(),
22730                recovered_workflow_task_count: 2,
22731            }
22732        );
22733    }
22734
22735    #[tokio::test]
22736    async fn low_level_registration_rejects_update_validators_before_transport() {
22737        let server = MockWorkerServer::start();
22738        let client = Client::builder(server.base_url())
22739            .timeout(Duration::from_secs(2))
22740            .build()
22741            .expect("client");
22742
22743        for update_validators in [json!(["approve"]), json!("approve")] {
22744            let error = client
22745                .register_worker_with_command_contracts(
22746                    "validator-claiming-worker",
22747                    "rust-workers",
22748                    vec!["orders".to_string()],
22749                    vec![],
22750                    1,
22751                    1,
22752                    vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22753                    json!({
22754                        "orders": {
22755                            "queries": ["current"],
22756                            "updates": ["approve"],
22757                            "update_validators": update_validators,
22758                        },
22759                    }),
22760                )
22761                .await
22762                .expect_err("unsupported validator claims must fail before registration");
22763
22764            let Error::UnsupportedUpdateValidators { workflow_type } = error else {
22765                panic!("expected typed unsupported-validator failure");
22766            };
22767            assert_eq!(workflow_type, "orders");
22768        }
22769        assert_eq!(server.request_count("/api/worker/register"), 0);
22770    }
22771
22772    #[tokio::test]
22773    async fn low_level_registration_preserves_query_and_update_contracts() {
22774        let server = MockWorkerServer::start();
22775        let client = Client::builder(server.base_url())
22776            .timeout(Duration::from_secs(2))
22777            .build()
22778            .expect("client");
22779        let contracts = json!({
22780            "orders": {
22781                "queries": ["current"],
22782                "updates": ["approve"],
22783                "update_validators": [],
22784            },
22785            "payments": {
22786                "queries": ["status"],
22787                "updates": ["capture"],
22788            },
22789        });
22790
22791        client
22792            .register_worker_with_command_contracts(
22793                "command-worker",
22794                "rust-workers",
22795                vec!["orders".to_string(), "payments".to_string()],
22796                vec![],
22797                1,
22798                1,
22799                vec![WORKFLOW_UPDATES_CAPABILITY.to_string()],
22800                contracts.clone(),
22801            )
22802            .await
22803            .expect("query and update contracts must remain supported");
22804
22805        assert_eq!(
22806            server.request_body("/api/worker/register")["workflow_command_contracts"],
22807            contracts
22808        );
22809    }
22810
22811    #[tokio::test]
22812    async fn role_scoped_tokens_are_never_used_for_the_opposite_plane() {
22813        let server = MockWorkerServer::start();
22814        let control_only = Client::builder(server.base_url())
22815            .control_token(Some("control-secret".to_string()))
22816            .build()
22817            .expect("control client");
22818
22819        let error = control_only
22820            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22821            .await
22822            .expect_err("control token must not authorize a worker request");
22823        assert!(matches!(
22824            error,
22825            Error::MissingRoleCredentials { role: "worker", .. }
22826        ));
22827        assert_eq!(server.request_count("/api/worker/register"), 0);
22828
22829        let worker_only = Client::builder(server.base_url())
22830            .worker_token(Some("worker-secret".to_string()))
22831            .build()
22832            .expect("worker client");
22833        let error = worker_only
22834            .health()
22835            .await
22836            .expect_err("worker token must not authorize a control request");
22837        assert!(matches!(
22838            error,
22839            Error::MissingRoleCredentials {
22840                role: "control",
22841                ..
22842            }
22843        ));
22844        assert_eq!(server.request_count("/api/health"), 0);
22845    }
22846
22847    #[tokio::test]
22848    async fn shared_token_supports_worker_and_control_planes() {
22849        let server = MockWorkerServer::start();
22850        let client = Client::builder(server.base_url())
22851            .token(Some("shared-secret".to_string()))
22852            .build()
22853            .expect("client");
22854
22855        client.health().await.expect("control request");
22856        client
22857            .register_worker("worker", "queue", vec![], vec![], 1, 1)
22858            .await
22859            .expect("worker request");
22860
22861        assert_eq!(
22862            server.authorization_for("/api/health").as_deref(),
22863            Some("Bearer shared-secret")
22864        );
22865        assert_eq!(
22866            server.control_protocol_for("/api/health").as_deref(),
22867            Some(CONTROL_PLANE_VERSION)
22868        );
22869        assert_eq!(
22870            server.authorization_for("/api/worker/register").as_deref(),
22871            Some("Bearer shared-secret")
22872        );
22873        assert_eq!(
22874            server
22875                .worker_protocol_for("/api/worker/register")
22876                .as_deref(),
22877            Some(WORKER_PROTOCOL_VERSION)
22878        );
22879    }
22880
22881    #[tokio::test]
22882    async fn baseline_worker_endpoints_send_the_baseline_protocol() {
22883        let server = MockWorkerServer::start();
22884        let client = Client::builder(server.base_url())
22885            .timeout(Duration::from_secs(2))
22886            .build()
22887            .expect("client");
22888
22889        client
22890            .register_worker("capture-worker", "capture", vec![], vec![], 1, 1)
22891            .await
22892            .expect("register");
22893        client
22894            .heartbeat_worker("capture-worker", 1, 1)
22895            .await
22896            .expect("heartbeat");
22897        client
22898            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
22899            .await
22900            .expect("workflow poll");
22901        client
22902            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
22903            .await
22904            .expect("activity poll");
22905
22906        for path in [
22907            "/api/worker/register",
22908            "/api/worker/heartbeat",
22909            "/api/worker/workflow-tasks/poll",
22910            "/api/worker/activity-tasks/poll",
22911        ] {
22912            assert_eq!(
22913                server.worker_protocol_for(path).as_deref(),
22914                Some(WORKER_PROTOCOL_VERSION),
22915                "unexpected protocol for {path}"
22916            );
22917        }
22918
22919        assert_eq!(
22920            server.request_body("/api/worker/workflow-tasks/poll")["timeout_seconds"],
22921            1
22922        );
22923        assert_eq!(
22924            server.request_body("/api/worker/activity-tasks/poll")["timeout_seconds"],
22925            1
22926        );
22927        assert!(
22928            server.request_body("/api/worker/workflow-tasks/poll")["poll_request_id"]
22929                .as_str()
22930                .is_some_and(|id| id.starts_with("rust-workflow-poll-"))
22931        );
22932        assert!(
22933            server.request_body("/api/worker/activity-tasks/poll")["poll_request_id"]
22934                .as_str()
22935                .is_some_and(|id| id.starts_with("rust-activity-poll-"))
22936        );
22937    }
22938
22939    #[tokio::test]
22940    async fn query_task_endpoints_send_the_query_feature_protocol() {
22941        let server = MockWorkerServer::start();
22942        let client = Client::builder(server.base_url())
22943            .timeout(Duration::from_secs(2))
22944            .build()
22945            .expect("client");
22946
22947        client
22948            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
22949            .await
22950            .expect("query poll");
22951        client
22952            .complete_query_task(
22953                "query-capture",
22954                "capture-worker",
22955                1,
22956                json!(8),
22957                DEFAULT_CODEC,
22958            )
22959            .await
22960            .expect("query complete");
22961        client
22962            .fail_query_task(
22963                "query-capture",
22964                "capture-worker",
22965                1,
22966                "failed",
22967                "query_rejected",
22968                "QueryFailed",
22969            )
22970            .await
22971            .expect("query fail");
22972
22973        for path in [
22974            "/api/worker/query-tasks/poll",
22975            "/api/worker/query-tasks/query-capture/complete",
22976            "/api/worker/query-tasks/query-capture/fail",
22977        ] {
22978            assert_eq!(
22979                server.worker_protocol_for(path).as_deref(),
22980                Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION),
22981                "unexpected protocol for {path}"
22982            );
22983        }
22984
22985        assert_eq!(
22986            server.request_body("/api/worker/query-tasks/poll")["timeout_seconds"],
22987            1
22988        );
22989        assert!(
22990            server.request_body("/api/worker/query-tasks/poll")["poll_request_id"]
22991                .as_str()
22992                .is_some_and(|id| id.starts_with("rust-query-poll-"))
22993        );
22994    }
22995
22996    #[tokio::test]
22997    async fn disconnected_client_polls_retry_once_with_the_same_request_id() {
22998        let server = MockWorkerServer::transient_worker_failures();
22999        let client = Client::builder(server.base_url())
23000            .timeout(Duration::from_secs(2))
23001            .build()
23002            .expect("client");
23003
23004        client
23005            .poll_workflow_task("capture-worker", "capture", Duration::from_millis(10))
23006            .await
23007            .expect("workflow poll retry");
23008        client
23009            .poll_activity_task("capture-worker", "capture", Duration::from_millis(10))
23010            .await
23011            .expect("activity poll retry");
23012        client
23013            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23014            .await
23015            .expect("query poll retry");
23016
23017        for path in [
23018            "/api/worker/workflow-tasks/poll",
23019            "/api/worker/activity-tasks/poll",
23020            "/api/worker/query-tasks/poll",
23021        ] {
23022            let bodies = server.request_bodies(path);
23023            assert_eq!(bodies.len(), 2, "{path} must be retried once");
23024            assert_eq!(
23025                bodies[0]["poll_request_id"], bodies[1]["poll_request_id"],
23026                "{path} must preserve the request binding across retry"
23027            );
23028        }
23029    }
23030
23031    #[tokio::test]
23032    async fn worker_poll_retries_preserve_request_id_across_consecutive_failures() {
23033        let server = MockWorkerServer::consecutive_poll_failures(2);
23034        let client = Client::builder(server.base_url())
23035            .timeout(Duration::from_secs(2))
23036            .build()
23037            .expect("client");
23038        let mut worker = Worker::new(client, "capture")
23039            .worker_id("capture-worker")
23040            .poll_timeout(Duration::from_millis(10))
23041            .retry_policy(WorkerRetryPolicy {
23042                max_retries: 2,
23043                initial_backoff: Duration::from_millis(1),
23044                max_backoff: Duration::from_millis(1),
23045            });
23046        worker.register_workflow(
23047            "capture.workflow",
23048            |_ctx, _input| async move { Ok(Value::Null) },
23049        );
23050        worker.register_activity(
23051            "capture.activity",
23052            |_ctx, _input| async move { Ok(Value::Null) },
23053        );
23054        worker.register_query("capture.workflow", "current", |_ctx, _args| async move {
23055            Ok(Value::Null)
23056        });
23057
23058        assert_eq!(worker.run_once().await.expect("poll retries"), 0);
23059
23060        for path in [
23061            "/api/worker/workflow-tasks/poll",
23062            "/api/worker/activity-tasks/poll",
23063            "/api/worker/query-tasks/poll",
23064        ] {
23065            let bodies = server.request_bodies(path);
23066            assert_eq!(bodies.len(), 3, "{path} must use exactly two retries");
23067            assert!(
23068                bodies
23069                    .iter()
23070                    .all(|body| body["poll_request_id"] == bodies[0]["poll_request_id"]),
23071                "{path} must preserve one request binding across every retry"
23072            );
23073        }
23074    }
23075
23076    fn storage_refusal(poll_id: Option<&str>, unavailable: bool, mid_poll: bool) -> Value {
23077        let reason = if unavailable {
23078            "storage_admission_unavailable"
23079        } else {
23080            "storage_pressure"
23081        };
23082        let mut body = json!({
23083            "reason": reason,
23084            "storage_state": if unavailable { "fenced" } else { "draining" },
23085            "retryable": true,
23086            "retry_after_seconds": 1,
23087        });
23088        if !mid_poll {
23089            body["request_admitted"] = json!(false);
23090        }
23091        if let Some(id) = poll_id {
23092            body["task"] = Value::Null;
23093            body["poll_status"] = json!(reason);
23094            body["poll_request_id"] = json!(id);
23095            body["retry_same_poll_request_id"] = json!(true);
23096            body["claim_admitted"] = json!(false);
23097        }
23098        body
23099    }
23100
23101    fn storage_worker(server: &MockWorkerServer) -> Worker {
23102        Worker::new(Client::new(server.base_url()).expect("client"), "storage")
23103            .worker_id("storage-worker")
23104            .retry_policy(WorkerRetryPolicy {
23105                max_retries: 1,
23106                initial_backoff: Duration::from_millis(1),
23107                max_backoff: Duration::from_millis(1),
23108            })
23109    }
23110
23111    fn assert_identical_requests(server: &MockWorkerServer, path: &str, count: usize) {
23112        let requests = server.requests.lock().expect("requests");
23113        let bodies: Vec<_> = requests
23114            .iter()
23115            .filter(|request| request.path == path)
23116            .map(|request| &request.body)
23117            .collect();
23118        assert_eq!(bodies.len(), count, "{path}");
23119        assert!(bodies.iter().all(|body| body == &bodies[0]), "{path}");
23120    }
23121
23122    #[test]
23123    fn storage_admission_requires_an_explicit_identity_preserving_contract() {
23124        for unavailable in [false, true] {
23125            for mid_poll in [false, true] {
23126                let body = storage_refusal(Some("same-poll"), unavailable, mid_poll);
23127                let error = Error::Http {
23128                    status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23129                    body: body.to_string(),
23130                };
23131                assert_eq!(
23132                    worker_storage_admission_retry_after(&error, Some("same-poll")),
23133                    Some(Duration::from_secs(1))
23134                );
23135                assert!(
23136                    !worker_operation_is_retryable(&error),
23137                    "storage is not a bounded generic retry"
23138                );
23139                for (field, value) in [
23140                    ("poll_request_id", json!("wrong-poll")),
23141                    ("task", json!({"task_id":"claimed"})),
23142                    ("retryable", json!(false)),
23143                    ("retry_after_seconds", json!(0)),
23144                    ("retry_after_seconds", json!(true)),
23145                    ("retry_after_seconds", json!(1.0)),
23146                    ("storage_state", json!("normal")),
23147                    ("poll_status", json!("empty")),
23148                    ("claim_admitted", json!(true)),
23149                    ("retry_same_poll_request_id", json!(false)),
23150                    ("request_admitted", json!(true)),
23151                ] {
23152                    let mut invalid = body.clone();
23153                    invalid[field] = value;
23154                    let error = Error::Http {
23155                        status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23156                        body: invalid.to_string(),
23157                    };
23158                    assert!(
23159                        worker_storage_admission_retry_after(&error, Some("same-poll")).is_none(),
23160                        "{field}"
23161                    );
23162                }
23163            }
23164        }
23165        let body = storage_refusal(None, false, false);
23166        let error = Error::Http {
23167            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23168            body: body.to_string(),
23169        };
23170        assert!(worker_storage_admission_retry_after(&error, None).is_some());
23171        assert!(worker_storage_admission_retry_after(&error, Some("")).is_none());
23172        let error = Error::Http {
23173            status: reqwest::StatusCode::FORBIDDEN,
23174            body: body.to_string(),
23175        };
23176        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23177        let body = storage_refusal(None, false, true);
23178        let error = Error::Http {
23179            status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23180            body: body.to_string(),
23181        };
23182        assert!(worker_storage_admission_retry_after(&error, None).is_none());
23183    }
23184
23185    #[tokio::test]
23186    async fn storage_poll_recovery_preserves_ambiguous_claim_identity() {
23187        for unavailable in [false, true] {
23188            for mid_poll in [false, true] {
23189                let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23190                    poll_failures_per_path: 1,
23191                    storage_refusals: 7,
23192                    storage_path: Some("/poll"),
23193                    storage_unavailable: unavailable,
23194                    storage_mid_poll: mid_poll,
23195                    ..MockWorkerBehavior::default()
23196                });
23197                let mut worker = storage_worker(&server);
23198                worker.register_query("unused", "state", |_, _| async { Ok(Value::Null) });
23199                assert_eq!(worker.run_once().await.expect("storage recovery"), 0);
23200                for path in [
23201                    "/api/worker/workflow-tasks/poll",
23202                    "/api/worker/activity-tasks/poll",
23203                    "/api/worker/query-tasks/poll",
23204                ] {
23205                    assert_identical_requests(&server, path, 9);
23206                }
23207            }
23208        }
23209    }
23210
23211    #[tokio::test]
23212    async fn storage_refused_mutations_do_not_reserialize_or_change_client_scope() {
23213        struct CountedBody(Arc<AtomicUsize>);
23214        impl Serialize for CountedBody {
23215            fn serialize<S: Serializer>(
23216                &self,
23217                serializer: S,
23218            ) -> std::result::Result<S::Ok, S::Error> {
23219                let count = self.0.fetch_add(1, Ordering::SeqCst);
23220                json!({"serialization":count,"lease_owner":"worker","attempt":7})
23221                    .serialize(serializer)
23222            }
23223        }
23224        for path in [
23225            "/api/worker/register",
23226            "/api/worker/heartbeat",
23227            "/api/worker/workflow-tasks/storage-task/complete",
23228            "/api/worker/workflow-tasks/storage-task/fail",
23229            "/api/worker/activity-tasks/storage-task/complete",
23230            "/api/worker/activity-tasks/storage-task/fail",
23231            "/api/worker/activity-tasks/storage-task/heartbeat",
23232            "/api/worker/query-tasks/storage-task/complete",
23233            "/api/worker/query-tasks/storage-task/fail",
23234        ] {
23235            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23236                storage_refusals: 7,
23237                storage_path: Some(path),
23238                ..MockWorkerBehavior::default()
23239            });
23240            let worker =
23241                storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23242            let calls = Arc::new(AtomicUsize::new(0));
23243            let _: Value = worker
23244                .client
23245                .request_json(
23246                    reqwest::Method::POST,
23247                    &path[4..],
23248                    RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23249                    Some(&CountedBody(Arc::clone(&calls))),
23250                )
23251                .await
23252                .expect("prepared request recovery");
23253            assert_eq!(calls.load(Ordering::SeqCst), 1);
23254            assert_identical_requests(&server, path, 8);
23255        }
23256        for worker_scope in [false, true] {
23257            let path = if worker_scope {
23258                "/api/health"
23259            } else {
23260                "/api/worker/register"
23261            };
23262            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23263                storage_refusals: usize::MAX,
23264                storage_path: Some(path),
23265                ..MockWorkerBehavior::default()
23266            });
23267            let worker = storage_worker(&server);
23268            let client = worker.client.clone();
23269            let worker = worker.with_storage_admission(Arc::new(AtomicBool::new(false)));
23270            let error = if worker_scope {
23271                worker
23272                    .client
23273                    .health()
23274                    .await
23275                    .expect_err("control plane is not retried")
23276            } else {
23277                client
23278                    .request_json::<Value, Value>(
23279                        reqwest::Method::POST,
23280                        "/worker/register",
23281                        RequestProtocol::Worker(WORKER_PROTOCOL_VERSION),
23282                        Some(&json!({})),
23283                    )
23284                    .await
23285                    .expect_err("direct client is not retried")
23286            };
23287            assert!(worker_storage_admission_body(&error).is_some());
23288            assert_eq!(server.request_count(path), 1);
23289        }
23290    }
23291
23292    #[tokio::test]
23293    async fn storage_activity_outcome_is_retained_without_reexecuting_handler() {
23294        for fail in [false, true] {
23295            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23296                storage_activity: true,
23297                storage_refusals: 7,
23298                storage_path: Some("/storage-activity/"),
23299                ..MockWorkerBehavior::default()
23300            });
23301            let mut worker = storage_worker(&server);
23302            let calls = Arc::new(AtomicUsize::new(0));
23303            let observed = Arc::clone(&calls);
23304            worker.register_activity("storage.activity", move |ctx, _| {
23305                let calls = Arc::clone(&observed);
23306                async move {
23307                    calls.fetch_add(1, Ordering::SeqCst);
23308                    ctx.heartbeat(json!({"step":2})).await?;
23309                    if fail {
23310                        Err(Error::WorkerLoop("intentional handler failure".to_string()))
23311                    } else {
23312                        Ok(json!({"receipt":true}))
23313                    }
23314                }
23315            });
23316            assert_eq!(worker.run_once().await.expect("activity settled"), 1);
23317            assert_eq!(calls.load(Ordering::SeqCst), 1);
23318            assert_identical_requests(
23319                &server,
23320                "/api/worker/activity-tasks/storage-activity/heartbeat",
23321                8,
23322            );
23323            let suffix = if fail { "fail" } else { "complete" };
23324            assert_identical_requests(
23325                &server,
23326                &format!("/api/worker/activity-tasks/storage-activity/{suffix}"),
23327                8,
23328            );
23329            let other = if fail { "complete" } else { "fail" };
23330            assert_eq!(
23331                server.request_count(&format!(
23332                    "/api/worker/activity-tasks/storage-activity/{other}"
23333                )),
23334                0
23335            );
23336        }
23337    }
23338
23339    #[tokio::test]
23340    async fn storage_waits_are_interruptible_without_false_activity_failure() {
23341        for path in [
23342            "/api/worker/register",
23343            "/api/worker/heartbeat",
23344            "/api/worker/activity-tasks/poll",
23345            "/api/worker/activity-tasks/storage-activity/heartbeat",
23346            "/api/worker/activity-tasks/storage-activity/complete",
23347        ] {
23348            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23349                storage_activity: true,
23350                storage_refusals: usize::MAX,
23351                storage_path: Some(path),
23352                ..MockWorkerBehavior::default()
23353            });
23354            let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23355            worker.register_activity("storage.activity", |ctx, _| async move {
23356                ctx.heartbeat(json!({"step":2})).await?;
23357                Ok(json!({"receipt":true}))
23358            });
23359            let shutdown = async {
23360                while server.request_count(path) == 0 {
23361                    tokio::time::sleep(Duration::from_millis(1)).await;
23362                }
23363            };
23364            let result = tokio::time::timeout(Duration::from_secs(2), worker.run_until(shutdown))
23365                .await
23366                .expect("shutdown interrupts admission");
23367            assert!(
23368                result.is_err(),
23369                "a refused operation must not appear acknowledged: {path}, {result:?}"
23370            );
23371            assert_eq!(server.request_count(path), 1);
23372            assert_eq!(
23373                server.request_count("/api/worker/activity-tasks/storage-activity/fail"),
23374                0
23375            );
23376            assert_eq!(
23377                server.request_count("/api/worker/registrations/mock-worker"),
23378                usize::from(!path.ends_with("/register"))
23379            );
23380        }
23381    }
23382
23383    #[tokio::test]
23384    async fn storage_query_outcome_is_retained_without_reexecuting_handler() {
23385        for fail in [false, true] {
23386            let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23387                storage_query: true,
23388                storage_refusals: 7,
23389                storage_path: Some("/storage-query/"),
23390                ..MockWorkerBehavior::default()
23391            });
23392            let mut worker = storage_worker(&server);
23393            worker.register_workflow("storage.workflow", |_, _| async { Ok(Value::Null) });
23394            let calls = Arc::new(AtomicUsize::new(0));
23395            let observed = Arc::clone(&calls);
23396            worker.register_query("storage.workflow", "state", move |_, _| {
23397                let calls = Arc::clone(&observed);
23398                async move {
23399                    calls.fetch_add(1, Ordering::SeqCst);
23400                    if fail {
23401                        Err(Error::WorkerLoop("intentional query failure".to_string()))
23402                    } else {
23403                        Ok(json!({"state":"waiting"}))
23404                    }
23405                }
23406            });
23407            assert_eq!(worker.run_once().await.expect("query settled"), 1);
23408            assert_eq!(calls.load(Ordering::SeqCst), 1);
23409            let suffix = if fail { "fail" } else { "complete" };
23410            assert_identical_requests(
23411                &server,
23412                &format!("/api/worker/query-tasks/storage-query/{suffix}"),
23413                8,
23414            );
23415            let other = if fail { "complete" } else { "fail" };
23416            assert_eq!(
23417                server.request_count(&format!("/api/worker/query-tasks/storage-query/{other}")),
23418                0
23419            );
23420        }
23421    }
23422
23423    #[tokio::test]
23424    async fn storage_recovery_does_not_override_auth_lease_or_invalid_contract() {
23425        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23426            storage_refusals: 7,
23427            storage_path: Some("/poll"),
23428            unauthorized_polls: true,
23429            ..MockWorkerBehavior::default()
23430        });
23431        let error = storage_worker(&server)
23432            .run_once()
23433            .await
23434            .expect_err("auth remains terminal");
23435        assert!(matches!(
23436            error,
23437            Error::Http {
23438                status: reqwest::StatusCode::UNAUTHORIZED,
23439                ..
23440            }
23441        ));
23442        assert_identical_requests(&server, "/api/worker/workflow-tasks/poll", 8);
23443
23444        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23445            storage_refusals: 7,
23446            storage_path: Some("/activity-cancel/complete"),
23447            ..MockWorkerBehavior::default()
23448        });
23449        let worker =
23450            storage_worker(&server).with_storage_admission(Arc::new(AtomicBool::new(false)));
23451        let error = worker
23452            .client
23453            .complete_activity_task(
23454                "activity-cancel",
23455                "attempt-cancel",
23456                "worker",
23457                json!({}),
23458                DEFAULT_CODEC,
23459            )
23460            .await
23461            .expect_err("cancellation remains terminal");
23462        assert!(activity_task_rejection_is_final(&error));
23463        assert_identical_requests(
23464            &server,
23465            "/api/worker/activity-tasks/activity-cancel/complete",
23466            8,
23467        );
23468
23469        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23470            storage_refusals: usize::MAX,
23471            storage_path: Some("/poll"),
23472            storage_wrong_poll_id: true,
23473            ..MockWorkerBehavior::default()
23474        });
23475        assert!(storage_worker(&server).run_once().await.is_err());
23476        assert_eq!(server.request_count("/api/worker/workflow-tasks/poll"), 1);
23477    }
23478
23479    #[tokio::test]
23480    async fn storage_pollers_stop_when_the_run_future_is_aborted() {
23481        let server = MockWorkerServer::start_with_behavior(MockWorkerBehavior {
23482            storage_refusals: usize::MAX,
23483            storage_path: Some("/poll"),
23484            ..MockWorkerBehavior::default()
23485        });
23486        let mut worker = storage_worker(&server).retry_policy(WorkerRetryPolicy::default());
23487        worker.register_activity("unused", |_, _| async { Ok(Value::Null) });
23488        let run = tokio::spawn(async move { worker.run().await });
23489        tokio::time::timeout(Duration::from_secs(2), async {
23490            while server.request_count("/api/worker/activity-tasks/poll") == 0 {
23491                tokio::time::sleep(Duration::from_millis(1)).await;
23492            }
23493        })
23494        .await
23495        .expect("poll started");
23496        run.abort();
23497        assert!(run.await.expect_err("cancelled run").is_cancelled());
23498        tokio::time::sleep(Duration::from_millis(250)).await;
23499        assert_eq!(server.request_count("/api/worker/activity-tasks/poll"), 1);
23500    }
23501
23502    #[tokio::test]
23503    async fn query_protocol_rejection_from_older_server_is_typed() {
23504        let server = MockWorkerServer::reject_query_protocol();
23505        let client = Client::builder(server.base_url())
23506            .timeout(Duration::from_secs(2))
23507            .build()
23508            .expect("client");
23509
23510        let error = client
23511            .poll_query_task("capture-worker", "capture", Duration::from_millis(10))
23512            .await
23513            .expect_err("server below query protocol floor must reject");
23514        let Error::Protocol(failure) = error else {
23515            panic!("expected typed protocol failure");
23516        };
23517
23518        assert_eq!(failure.status, 400);
23519        assert_eq!(failure.reason, "unsupported_protocol_version");
23520        assert_eq!(failure.supported_version.as_deref(), Some("1.7"));
23521        assert_eq!(
23522            failure.requested_version.as_deref(),
23523            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23524        );
23525        assert_eq!(
23526            server
23527                .worker_protocol_for("/api/worker/query-tasks/poll")
23528                .as_deref(),
23529            Some(QUERY_TASK_MINIMUM_WORKER_PROTOCOL_VERSION)
23530        );
23531    }
23532
23533    #[tokio::test]
23534    async fn run_once_without_query_handlers_keeps_pre_query_server_compatibility() {
23535        let server = MockWorkerServer::reject_query_protocol();
23536        let client = Client::builder(server.base_url())
23537            .timeout(Duration::from_secs(2))
23538            .build()
23539            .expect("client");
23540        let mut worker = Worker::new(client, "rust-workers")
23541            .worker_id("baseline-worker")
23542            .poll_timeout(Duration::from_millis(10));
23543
23544        worker.register_workflow("baseline.workflow", |_ctx, _input| async move {
23545            Ok(Value::Null)
23546        });
23547
23548        assert_eq!(worker.run_once().await.expect("baseline run once"), 0);
23549        assert_eq!(
23550            server
23551                .worker_protocol_for("/api/worker/workflow-tasks/poll")
23552                .as_deref(),
23553            Some(WORKER_PROTOCOL_VERSION)
23554        );
23555        assert_eq!(
23556            server.worker_protocol_for("/api/worker/query-tasks/poll"),
23557            None,
23558            "a worker without query handlers must not use the query-task endpoint"
23559        );
23560    }
23561
23562    #[tokio::test]
23563    async fn completion_time_query_rejection_is_typed_without_stopping_worker() {
23564        let server = MockWorkerServer::reject_query_completion();
23565        let client = Client::builder(server.base_url())
23566            .timeout(Duration::from_secs(2))
23567            .build()
23568            .expect("client");
23569
23570        let error = client
23571            .complete_query_task("query-late", "late-worker", 1, json!(8), DEFAULT_CODEC)
23572            .await
23573            .expect_err("expired completion must be rejected");
23574        let Error::QueryFailed(failure) = error else {
23575            panic!("expected typed query failure");
23576        };
23577        assert_eq!(failure.status, 409);
23578        assert_eq!(failure.reason, "query_task_timed_out");
23579
23580        let mut worker = Worker::new(client, "rust-workers")
23581            .worker_id("late-worker")
23582            .poll_timeout(Duration::from_millis(10));
23583        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
23584        worker.register_query(
23585            "counter",
23586            "current",
23587            |_ctx, _args| async move { Ok(json!(8)) },
23588        );
23589
23590        assert_eq!(worker.run_once().await.expect("late task is handled"), 1);
23591        assert_eq!(
23592            worker
23593                .run_once()
23594                .await
23595                .expect("worker continues after late completion"),
23596            0
23597        );
23598        assert_eq!(
23599            server.request_count("/api/worker/query-tasks/query-late/complete"),
23600            2
23601        );
23602        assert_eq!(
23603            server.request_count("/api/worker/query-tasks/query-late/fail"),
23604            0,
23605            "a server completion rejection must not be reported as an encoding failure"
23606        );
23607    }
23608
23609    #[tokio::test]
23610    async fn normal_shutdown_joins_pollers_and_deregisters_once() {
23611        let server = MockWorkerServer::start();
23612        let client = Client::builder(server.base_url())
23613            .timeout(Duration::from_secs(2))
23614            .build()
23615            .expect("client");
23616        let mut worker = Worker::new(client, "rust-workers")
23617            .worker_id("joined-worker")
23618            .poll_timeout(Duration::from_millis(10));
23619        worker.register_workflow(
23620            "joined.workflow",
23621            |_ctx, _input| async move { Ok(Value::Null) },
23622        );
23623        worker.register_activity(
23624            "joined.activity",
23625            |_ctx, _input| async move { Ok(Value::Null) },
23626        );
23627        worker.register_query("joined.workflow", "state", |_ctx, _input| async move {
23628            Ok(Value::Null)
23629        });
23630
23631        worker
23632            .run_until(tokio::time::sleep(Duration::from_millis(20)))
23633            .await
23634            .expect("normal shutdown");
23635
23636        let deregistration_path = "/api/worker/registrations/mock-worker";
23637        assert_eq!(server.request_count(deregistration_path), 1);
23638        for poll_path in [
23639            "/api/worker/workflow-tasks/poll",
23640            "/api/worker/activity-tasks/poll",
23641            "/api/worker/query-tasks/poll",
23642        ] {
23643            assert!(server.request_count(poll_path) > 0, "missing {poll_path}");
23644        }
23645        assert_eq!(
23646            server.captured_paths().last().map(String::as_str),
23647            Some(deregistration_path),
23648            "deregistration must start only after every poller has joined"
23649        );
23650    }
23651
23652    #[tokio::test]
23653    async fn registration_failure_does_not_deregister() {
23654        let server = MockWorkerServer::rejected_registration();
23655        let client = Client::builder(server.base_url())
23656            .timeout(Duration::from_secs(2))
23657            .build()
23658            .expect("client");
23659        let worker = Worker::new(client, "rust-workers").worker_id("never-registered");
23660
23661        let error = worker
23662            .run_until(async {})
23663            .await
23664            .expect_err("registration must fail");
23665        assert!(matches!(
23666            error,
23667            Error::Http {
23668                status: reqwest::StatusCode::SERVICE_UNAVAILABLE,
23669                ..
23670            }
23671        ));
23672        assert!(server
23673            .captured_paths()
23674            .iter()
23675            .all(|path| !path.starts_with("/api/worker/registrations/")));
23676    }
23677
23678    #[tokio::test]
23679    async fn protocol_116_server_rejects_occurrence_identity_worker_registration() {
23680        let server = MockWorkerServer::rejected_registration_protocol();
23681        let client = Client::builder(server.base_url())
23682            .timeout(Duration::from_secs(2))
23683            .build()
23684            .expect("client");
23685        let worker = Worker::new(client, "rust-workers").worker_id("protocol-117-worker");
23686
23687        let error = worker
23688            .run_until(async {})
23689            .await
23690            .expect_err("a protocol 1.16 server must reject this worker");
23691        let Error::Protocol(failure) = error else {
23692            panic!("expected typed protocol rejection");
23693        };
23694        assert_eq!(failure.reason, "unsupported_protocol_version");
23695        assert_eq!(failure.supported_version.as_deref(), Some("1.16"));
23696        assert_eq!(failure.requested_version.as_deref(), Some("1.17"));
23697        assert_eq!(
23698            server
23699                .worker_protocol_for("/api/worker/register")
23700                .as_deref(),
23701            Some(WORKER_PROTOCOL_VERSION)
23702        );
23703    }
23704
23705    #[tokio::test]
23706    async fn declined_registration_does_not_deregister() {
23707        let server = MockWorkerServer::declined_registration();
23708        let client = Client::builder(server.base_url())
23709            .timeout(Duration::from_secs(2))
23710            .build()
23711            .expect("client");
23712        let worker = Worker::new(client, "rust-workers").worker_id("declined-worker");
23713
23714        let error = worker
23715            .run_until(async {})
23716            .await
23717            .expect_err("declined registration must fail");
23718        assert!(matches!(error, Error::WorkerLoop(_)));
23719        assert!(error.to_string().contains("was not accepted"));
23720        assert!(server
23721            .captured_paths()
23722            .iter()
23723            .all(|path| !path.starts_with("/api/worker/registrations/")));
23724    }
23725
23726    #[tokio::test]
23727    async fn deregistration_http_failure_is_returned_after_normal_shutdown() {
23728        let server = MockWorkerServer::rejected_deregistration();
23729        let client = Client::builder(server.base_url())
23730            .timeout(Duration::from_secs(2))
23731            .build()
23732            .expect("client");
23733        let worker = Worker::new(client, "rust-workers").worker_id("forbidden-cleanup");
23734
23735        let error = worker
23736            .run_until(async {})
23737            .await
23738            .expect_err("deregistration must fail");
23739        assert!(matches!(
23740            error,
23741            Error::Http {
23742                status: reqwest::StatusCode::FORBIDDEN,
23743                ..
23744            }
23745        ));
23746        assert_eq!(
23747            server.request_count("/api/worker/registrations/mock-worker"),
23748            1
23749        );
23750    }
23751
23752    #[tokio::test]
23753    async fn deregistration_protocol_failure_is_returned_after_normal_shutdown() {
23754        let server = MockWorkerServer::rejected_deregistration_protocol();
23755        let client = Client::builder(server.base_url())
23756            .timeout(Duration::from_secs(2))
23757            .build()
23758            .expect("client");
23759        let worker = Worker::new(client, "rust-workers").worker_id("protocol-cleanup");
23760
23761        let error = worker
23762            .run_until(async {})
23763            .await
23764            .expect_err("protocol rejection must fail shutdown");
23765        let Error::Protocol(failure) = error else {
23766            panic!("expected typed protocol failure");
23767        };
23768        assert_eq!(failure.reason, "unsupported_protocol_version");
23769        assert_eq!(
23770            failure.requested_version.as_deref(),
23771            Some(WORKER_PROTOCOL_VERSION)
23772        );
23773        assert_eq!(
23774            server.request_count("/api/worker/registrations/mock-worker"),
23775            1
23776        );
23777    }
23778
23779    #[tokio::test]
23780    async fn primary_poller_error_retains_deregistration_failure_context() {
23781        let server = MockWorkerServer::unauthorized_polls_and_rejected_deregistration();
23782        let client = Client::builder(server.base_url())
23783            .timeout(Duration::from_secs(2))
23784            .build()
23785            .expect("client");
23786        let mut worker = Worker::new(client, "rust-workers")
23787            .worker_id("combined-failure")
23788            .poll_timeout(Duration::from_millis(10));
23789        worker.register_workflow("combined.workflow", |_ctx, _input| async move {
23790            Ok(Value::Null)
23791        });
23792
23793        let error = worker
23794            .run()
23795            .await
23796            .expect_err("worker and cleanup must fail");
23797        let summary = error.to_string();
23798        assert!(summary.contains("authentication_failed"));
23799        assert!(summary.contains("worker cannot deregister"));
23800        let Error::WorkerShutdown {
23801            primary,
23802            deregistration,
23803        } = error
23804        else {
23805            panic!("expected combined worker shutdown error");
23806        };
23807        assert!(matches!(
23808            *primary,
23809            Error::Http {
23810                status: reqwest::StatusCode::UNAUTHORIZED,
23811                ..
23812            }
23813        ));
23814        assert!(matches!(
23815            *deregistration,
23816            Error::Http {
23817                status: reqwest::StatusCode::FORBIDDEN,
23818                ..
23819            }
23820        ));
23821        assert_eq!(
23822            server.request_count("/api/worker/registrations/mock-worker"),
23823            1
23824        );
23825    }
23826
23827    #[tokio::test]
23828    async fn activity_only_worker_can_shutdown_without_workflow_poller() {
23829        let server = MockWorkerServer::start();
23830        let client = Client::builder(server.base_url())
23831            .timeout(Duration::from_secs(2))
23832            .build()
23833            .expect("client");
23834        let mut worker = Worker::new(client, "rust-workers")
23835            .worker_id("activity-only-worker")
23836            .poll_timeout(Duration::from_millis(10));
23837
23838        worker.register_activity(
23839            "activity.only",
23840            |_ctx, _args| async move { Ok(Value::Null) },
23841        );
23842
23843        worker.run_until(async {}).await.expect("run worker");
23844    }
23845
23846    #[tokio::test]
23847    async fn workflow_only_worker_can_shutdown_without_activity_poller() {
23848        let server = MockWorkerServer::start();
23849        let client = Client::builder(server.base_url())
23850            .timeout(Duration::from_secs(2))
23851            .build()
23852            .expect("client");
23853        let mut worker = Worker::new(client, "rust-workers")
23854            .worker_id("workflow-only-worker")
23855            .poll_timeout(Duration::from_millis(10));
23856
23857        worker.register_workflow(
23858            "workflow.only",
23859            |_ctx, _input| async move { Ok(Value::Null) },
23860        );
23861
23862        worker.run_until(async {}).await.expect("run worker");
23863    }
23864
23865    #[tokio::test]
23866    async fn worker_heartbeat_observer_receives_server_acknowledgements() {
23867        let server = MockWorkerServer::start();
23868        let client = Client::builder(server.base_url())
23869            .timeout(Duration::from_secs(2))
23870            .build()
23871            .expect("client");
23872        let observations = Arc::new(Mutex::new(Vec::new()));
23873        let observed = Arc::clone(&observations);
23874        let mut worker = Worker::new(client, "rust-workers")
23875            .worker_id("observed-heartbeat-worker")
23876            .poll_timeout(Duration::from_millis(10))
23877            .on_worker_heartbeat(move |observation| {
23878                observed
23879                    .lock()
23880                    .expect("heartbeat observations")
23881                    .push(observation.clone());
23882            });
23883
23884        worker.register_workflow("workflow.observed", |_ctx, _input| async move {
23885            Ok(Value::Null)
23886        });
23887        let acknowledged = Arc::clone(&observations);
23888        worker
23889            .run_until(async move {
23890                tokio::time::timeout(Duration::from_secs(2), async move {
23891                    loop {
23892                        if !acknowledged
23893                            .lock()
23894                            .expect("heartbeat observations")
23895                            .is_empty()
23896                        {
23897                            break;
23898                        }
23899                        tokio::time::sleep(Duration::from_millis(1)).await;
23900                    }
23901                })
23902                .await
23903                .expect("heartbeat acknowledgement within timeout");
23904            })
23905            .await
23906            .expect("run worker");
23907
23908        let observations = observations.lock().expect("heartbeat observations");
23909        let first = observations.first().expect("heartbeat acknowledgement");
23910        assert_eq!(first.worker_id, "observed-heartbeat-worker");
23911        assert_eq!(first.task_queue, "rust-workers");
23912        assert!(first.acknowledged_at_unix_millis > 0);
23913        assert_eq!(first.acknowledgement, json!({}));
23914    }
23915
23916    #[tokio::test]
23917    async fn delayed_worker_heartbeat_keeps_cadence_and_pollers_live() {
23918        let server = MockWorkerServer::delayed_heartbeat_worker();
23919        let client = Client::builder(server.base_url())
23920            .timeout(Duration::from_secs(3))
23921            .build()
23922            .expect("client");
23923        let observations = Arc::new(Mutex::new(Vec::new()));
23924        let observed = Arc::clone(&observations);
23925        let mut worker = Worker::new(client, "rust-snapshot-workers")
23926            .worker_id("rust-snapshot-worker")
23927            .poll_timeout(Duration::from_millis(10))
23928            .on_worker_heartbeat(move |observation| {
23929                observed
23930                    .lock()
23931                    .expect("heartbeat observations")
23932                    .push(observation.clone());
23933            });
23934
23935        worker.register_workflow("snapshot", |ctx, _input| async move {
23936            ctx.wait_signal("finish").await?;
23937            Ok(json!({"status": "finished"}))
23938        });
23939        worker.register_query("snapshot", "current", |ctx, _args| async move {
23940            Ok(json!(ctx
23941                .signals("increment")
23942                .iter()
23943                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
23944                .sum::<i64>()))
23945        });
23946        worker.register_activity("cancel-aware", |_ctx, _args| async move {
23947            Ok(json!({"late": "completion"}))
23948        });
23949
23950        worker
23951            .run_until(tokio::time::sleep(Duration::from_millis(3_800)))
23952            .await
23953            .expect("delayed heartbeat must allow a clean worker shutdown");
23954
23955        let observations = observations.lock().expect("heartbeat observations");
23956        assert!(
23957            observations.len() >= 3,
23958            "the immediate heartbeat, delayed acknowledgement, and next cadence heartbeat must complete"
23959        );
23960        assert!(
23961            observations.windows(2).all(|pair| {
23962                pair[1].acknowledged_at_unix_millis
23963                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
23964                    >= 850
23965            }),
23966            "successful acknowledgements must not catch up faster than the advertised one-second cadence: {observations:?}"
23967        );
23968        drop(observations);
23969
23970        let heartbeat_times = server.request_times("/api/worker/heartbeat");
23971        let delayed_request_at = *heartbeat_times
23972            .get(1)
23973            .expect("intentionally delayed heartbeat request");
23974        let delay_window_start = delayed_request_at + Duration::from_millis(100);
23975        let delay_window_end = delayed_request_at + Duration::from_millis(1_400);
23976        for path in [
23977            "/api/worker/workflow-tasks/poll",
23978            "/api/worker/activity-tasks/poll",
23979            "/api/worker/query-tasks/poll",
23980        ] {
23981            assert!(
23982                server
23983                    .request_times(path)
23984                    .iter()
23985                    .any(|received_at| *received_at >= delay_window_start
23986                        && *received_at <= delay_window_end),
23987                "{path} must keep polling while a heartbeat acknowledgement is delayed"
23988            );
23989        }
23990        assert!(
23991            server.request_count("/api/worker/workflow-tasks/snapshot-wait-3/fail") >= 1,
23992            "workflow work must be settled"
23993        );
23994        assert!(
23995            server.request_count("/api/worker/activity-tasks/activity-cancel/complete") >= 1,
23996            "activity work must be settled"
23997        );
23998        assert!(
23999            server.request_count("/api/worker/query-tasks/snapshot-current/complete") >= 1,
24000            "query work must be settled"
24001        );
24002    }
24003
24004    #[tokio::test]
24005    async fn retried_worker_heartbeat_restarts_the_advertised_cadence() {
24006        let server = MockWorkerServer::heartbeat_retry_worker();
24007        let client = Client::builder(server.base_url())
24008            .timeout(Duration::from_secs(2))
24009            .build()
24010            .expect("client");
24011        let observations = Arc::new(Mutex::new(Vec::new()));
24012        let observed = Arc::clone(&observations);
24013        let worker = Worker::new(client, "rust-workers")
24014            .worker_id("heartbeat-retry-worker")
24015            .retry_policy(WorkerRetryPolicy {
24016                max_retries: 1,
24017                initial_backoff: Duration::from_millis(300),
24018                max_backoff: Duration::from_millis(300),
24019            })
24020            .on_worker_heartbeat(move |observation| {
24021                observed
24022                    .lock()
24023                    .expect("heartbeat observations")
24024                    .push(observation.clone());
24025            });
24026
24027        worker
24028            .run_until(tokio::time::sleep(Duration::from_millis(2_700)))
24029            .await
24030            .expect("retryable heartbeat failure must remain bounded and recover");
24031
24032        let observations = observations.lock().expect("heartbeat observations");
24033        assert!(observations.len() >= 3, "heartbeat retry must recover");
24034        assert!(
24035            observations.windows(2).all(|pair| {
24036                pair[1]
24037                    .acknowledged_at_unix_millis
24038                    .saturating_sub(pair[0].acknowledged_at_unix_millis)
24039                    >= 850
24040            }),
24041            "a successful retry must start a fresh advertised cadence: {observations:?}"
24042        );
24043        assert_eq!(
24044            server.request_count("/api/worker/heartbeat"),
24045            observations.len() + 1,
24046            "one retryable failure must add exactly one bounded request"
24047        );
24048    }
24049
24050    #[tokio::test]
24051    async fn query_enabled_worker_ignores_unmatched_signals_then_completes_once() {
24052        let server = MockWorkerServer::waiting_query_worker();
24053        let client = Client::builder(server.base_url())
24054            .timeout(Duration::from_secs(2))
24055            .build()
24056            .expect("client");
24057        let observations = Arc::new(Mutex::new(Vec::new()));
24058        let observed = Arc::clone(&observations);
24059        let mut worker = Worker::new(client, "rust-snapshot-workers")
24060            .worker_id("rust-snapshot-worker")
24061            .poll_timeout(Duration::from_millis(10))
24062            .on_worker_heartbeat(move |observation| {
24063                observed
24064                    .lock()
24065                    .expect("heartbeat observations")
24066                    .push(observation.clone());
24067            });
24068
24069        worker.register_workflow("snapshot", |ctx, _input| async move {
24070            ctx.wait_signal("finish").await?;
24071            Ok(json!({"status": "finished"}))
24072        });
24073        worker.register_query("snapshot", "current", |ctx, _args| async move {
24074            let current = ctx
24075                .signals("increment")
24076                .iter()
24077                .filter_map(|arguments| arguments.first().and_then(Value::as_i64))
24078                .sum::<i64>();
24079            Ok(json!(current))
24080        });
24081        worker.register_update("snapshot", "replace", |_ctx, args| async move { Ok(args) });
24082
24083        worker
24084            .run_until(tokio::time::sleep(Duration::from_millis(3_200)))
24085            .await
24086            .expect("pending workflow and query poller must remain live until shutdown");
24087
24088        assert!(
24089            observations.lock().expect("heartbeat observations").len() >= 4,
24090            "the immediate heartbeat and at least three advertised one-second intervals must be acknowledged"
24091        );
24092        assert!(
24093            server.request_count("/api/worker/workflow-tasks/poll") >= 3,
24094            "workflow polling must continue after empty replay acknowledgements"
24095        );
24096        assert!(
24097            server.request_count("/api/worker/query-tasks/poll") >= 2,
24098            "query polling must continue after serving the current query"
24099        );
24100        assert_eq!(
24101            server.request_body("/api/worker/register")["capabilities"],
24102            json!([
24103                CONDITION_WAIT_OCCURRENCE_IDENTITY_CAPABILITY,
24104                DURABLE_SELECTION_CAPABILITY,
24105                MEMO_UPSERTS_CAPABILITY,
24106                TYPED_SEARCH_ATTRIBUTES_CAPABILITY,
24107                QUERY_TASKS_CAPABILITY,
24108                WORKFLOW_UPDATES_CAPABILITY,
24109                MESSAGE_STREAMS_CAPABILITY
24110            ])
24111        );
24112        assert_eq!(
24113            server.request_body("/api/worker/register")["workflow_command_contracts"]["snapshot"],
24114            json!({
24115                "queries": ["current"],
24116                "query_contracts": [],
24117                "signals": [],
24118                "signal_contracts": [],
24119                "updates": ["replace"],
24120                "update_contracts": [],
24121                "update_validators": [],
24122            })
24123        );
24124
24125        let opened = server.request_body("/api/worker/workflow-tasks/snapshot-open/complete");
24126        assert_eq!(
24127            opened["commands"],
24128            json!([{
24129                "type": "open_signal_wait",
24130                "signal_name": "finish",
24131            }])
24132        );
24133
24134        for task_id in ["snapshot-wait-3", "snapshot-wait-5"] {
24135            let fail_path = format!("/api/worker/workflow-tasks/{task_id}/fail");
24136            let completion_path = format!("/api/worker/workflow-tasks/{task_id}/complete");
24137            let failure = server.request_body(&fail_path);
24138            assert_eq!(
24139                failure["failure"]["type"],
24140                WORKFLOW_TASK_WAITING_FOR_HISTORY_TYPE
24141            );
24142            assert_eq!(server.request_count(&completion_path), 0);
24143        }
24144
24145        let query_completion =
24146            server.request_body("/api/worker/query-tasks/snapshot-current/complete");
24147        assert_eq!(query_completion["result"], json!(8));
24148
24149        let terminal_path = "/api/worker/workflow-tasks/snapshot-finish/complete";
24150        assert_eq!(
24151            server.request_count(terminal_path),
24152            1,
24153            "the matching signal must settle the workflow exactly once"
24154        );
24155        let terminal = server.request_body(terminal_path);
24156        assert_eq!(terminal["commands"].as_array().map(Vec::len), Some(1));
24157        assert_eq!(terminal["commands"][0]["type"], "complete_workflow");
24158        assert_eq!(
24159            decode_wire_value(&terminal["commands"][0]["result"], DEFAULT_CODEC)
24160                .expect("terminal workflow result"),
24161            json!({"status": "finished"})
24162        );
24163    }
24164
24165    #[tokio::test]
24166    async fn worker_retries_poll_and_heartbeat_transport_failures_independently() {
24167        let server = MockWorkerServer::transient_worker_failures();
24168        let client = Client::builder(server.base_url())
24169            .timeout(Duration::from_secs(2))
24170            .build()
24171            .expect("client");
24172        let mut worker = Worker::new(client, "rust-workers")
24173            .worker_id("retry-worker")
24174            .poll_timeout(Duration::from_millis(10))
24175            .retry_policy(WorkerRetryPolicy {
24176                max_retries: 2,
24177                initial_backoff: Duration::from_millis(1),
24178                max_backoff: Duration::from_millis(1),
24179            });
24180        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24181        worker.register_activity(
24182            "counter.activity",
24183            |_ctx, _input| async move { Ok(Value::Null) },
24184        );
24185        worker.register_query(
24186            "counter",
24187            "current",
24188            |_ctx, _args| async move { Ok(json!(8)) },
24189        );
24190
24191        worker
24192            .run_until(tokio::time::sleep(Duration::from_millis(75)))
24193            .await
24194            .expect("transient failures must not stop the worker");
24195
24196        for path in [
24197            "/api/worker/heartbeat",
24198            "/api/worker/workflow-tasks/poll",
24199            "/api/worker/activity-tasks/poll",
24200            "/api/worker/query-tasks/poll",
24201        ] {
24202            assert!(
24203                server.request_count(path) >= 2,
24204                "{path} must continue after its transient failure"
24205            );
24206        }
24207    }
24208
24209    #[tokio::test]
24210    async fn worker_continues_after_long_poll_capacity_backpressure() {
24211        let server = MockWorkerServer::capacity_limited_activity_poll();
24212        let client = Client::builder(server.base_url())
24213            .timeout(Duration::from_secs(2))
24214            .build()
24215            .expect("client");
24216        let mut worker = Worker::new(client, "rust-workers")
24217            .worker_id("capacity-worker")
24218            .poll_timeout(Duration::from_millis(10))
24219            .retry_policy(WorkerRetryPolicy {
24220                max_retries: 0,
24221                initial_backoff: Duration::from_millis(1),
24222                max_backoff: Duration::from_millis(1),
24223            });
24224        worker.register_activity("capacity.activity", |_ctx, _input| async move {
24225            Ok(json!({"handled": true}))
24226        });
24227
24228        worker
24229            .run_until(tokio::time::sleep(Duration::from_millis(50)))
24230            .await
24231            .expect("capacity backpressure must not stop the worker");
24232
24233        assert!(
24234            server.request_count("/api/worker/activity-tasks/poll") >= 2,
24235            "the activity poller must continue after capacity backpressure"
24236        );
24237        assert_eq!(
24238            server.request_count("/api/worker/activity-tasks/capacity-activity/complete"),
24239            1,
24240            "the worker must complete work returned after capacity recovers"
24241        );
24242    }
24243
24244    #[test]
24245    fn worker_poll_capacity_backpressure_requires_the_typed_retryable_contract() {
24246        let capacity = Error::Http {
24247            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24248            body: r#"{"poll_status":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":3}"#.to_string(),
24249        };
24250        assert_eq!(
24251            worker_poll_capacity_retry_after(&capacity),
24252            Some(Duration::from_secs(3))
24253        );
24254
24255        let rejected_capacity = Error::Http {
24256            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24257            body: r#"{"reason":"long_poll_capacity_exhausted","retryable":false,"retry_after_seconds":3}"#.to_string(),
24258        };
24259        assert_eq!(worker_poll_capacity_retry_after(&rejected_capacity), None);
24260        assert!(!worker_operation_is_retryable(&rejected_capacity));
24261
24262        let ordinary_rate_limit = Error::Http {
24263            status: reqwest::StatusCode::TOO_MANY_REQUESTS,
24264            body: r#"{"reason":"rate_limited","retryable":true,"retry_after_seconds":3}"#
24265                .to_string(),
24266        };
24267        assert_eq!(worker_poll_capacity_retry_after(&ordinary_rate_limit), None);
24268        assert!(worker_operation_is_retryable(&ordinary_rate_limit));
24269    }
24270
24271    #[tokio::test]
24272    async fn worker_bounds_transport_retries() {
24273        let server = MockWorkerServer::unavailable_polls();
24274        let client = Client::builder(server.base_url())
24275            .timeout(Duration::from_secs(2))
24276            .build()
24277            .expect("client");
24278        let mut worker = Worker::new(client, "rust-workers")
24279            .worker_id("bounded-retry-worker")
24280            .poll_timeout(Duration::from_millis(10))
24281            .retry_policy(WorkerRetryPolicy {
24282                max_retries: 2,
24283                initial_backoff: Duration::from_millis(1),
24284                max_backoff: Duration::from_millis(1),
24285            });
24286        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24287
24288        let error = worker.run().await.expect_err("retry bound must terminate");
24289        assert!(matches!(error, Error::Transport(_)));
24290        assert_eq!(
24291            server.request_count("/api/worker/workflow-tasks/poll"),
24292            3,
24293            "one initial request plus exactly two retries"
24294        );
24295    }
24296
24297    #[tokio::test]
24298    async fn worker_retry_policy_can_disable_poll_retries() {
24299        let server = MockWorkerServer::unavailable_polls();
24300        let client = Client::builder(server.base_url())
24301            .timeout(Duration::from_secs(2))
24302            .build()
24303            .expect("client");
24304        let mut worker = Worker::new(client, "rust-workers")
24305            .worker_id("no-retry-worker")
24306            .poll_timeout(Duration::from_millis(10))
24307            .retry_policy(WorkerRetryPolicy {
24308                max_retries: 0,
24309                initial_backoff: Duration::from_millis(1),
24310                max_backoff: Duration::from_millis(1),
24311            });
24312        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24313
24314        let error = worker
24315            .run_once()
24316            .await
24317            .expect_err("disabled retries must return the first transport failure");
24318        assert!(matches!(error, Error::Transport(_)));
24319        assert_eq!(
24320            server.request_count("/api/worker/workflow-tasks/poll"),
24321            1,
24322            "max_retries=0 must send only the initial request"
24323        );
24324    }
24325
24326    #[tokio::test]
24327    async fn worker_does_not_retry_authentication_failures() {
24328        let server = MockWorkerServer::unauthorized_polls();
24329        let client = Client::builder(server.base_url())
24330            .timeout(Duration::from_secs(2))
24331            .build()
24332            .expect("client");
24333        let mut worker = Worker::new(client, "rust-workers")
24334            .worker_id("unauthorized-worker")
24335            .poll_timeout(Duration::from_millis(10));
24336        worker.register_workflow("counter", |_ctx, _input| async move { Ok(Value::Null) });
24337
24338        let error = worker
24339            .run()
24340            .await
24341            .expect_err("authentication must terminate");
24342        let Error::Http { status, body } = error else {
24343            panic!("expected stable HTTP authentication error");
24344        };
24345        assert_eq!(status, reqwest::StatusCode::UNAUTHORIZED);
24346        assert!(body.contains("authentication_failed"));
24347        assert_eq!(
24348            server.request_count("/api/worker/workflow-tasks/poll"),
24349            1,
24350            "authentication failures must not be retried"
24351        );
24352    }
24353
24354    #[derive(Clone, Debug)]
24355    struct CapturedRequest {
24356        headers: String,
24357        method: String,
24358        path: String,
24359        authorization: Option<String>,
24360        namespace: Option<String>,
24361        worker_protocol: Option<String>,
24362        control_protocol: Option<String>,
24363        body: String,
24364        received_at: Instant,
24365    }
24366
24367    struct MockWorkerServer {
24368        addr: SocketAddr,
24369        stop: Arc<AtomicBool>,
24370        requests: Arc<Mutex<Vec<CapturedRequest>>>,
24371        thread: Option<thread::JoinHandle<()>>,
24372    }
24373
24374    type RequestOverride = fn(&str, &str, usize) -> Option<(&'static str, String)>;
24375
24376    #[derive(Clone, Copy, Default)]
24377    struct MockWorkerBehavior {
24378        response_override: Option<fn(&str) -> Option<(&'static str, String)>>,
24379        request_override: Option<RequestOverride>,
24380        storage_refusals: usize,
24381        storage_path: Option<&'static str>,
24382        storage_unavailable: bool,
24383        storage_mid_poll: bool,
24384        storage_activity: bool,
24385        storage_query: bool,
24386        storage_wrong_poll_id: bool,
24387        reject_query_protocol: bool,
24388        reject_query_completion: bool,
24389        waiting_query_worker: bool,
24390        decline_registration: bool,
24391        complete_named_signal: bool,
24392        poll_failures_per_path: usize,
24393        long_poll_capacity_responses_per_path: usize,
24394        heartbeat_failures: usize,
24395        heartbeat_failure_request: Option<usize>,
24396        delayed_heartbeat_request: Option<usize>,
24397        heartbeat_response_delay: Duration,
24398        concurrent_requests: bool,
24399        unauthorized_polls: bool,
24400        reject_registration: bool,
24401        reject_registration_protocol: bool,
24402        reject_deregistration: bool,
24403        reject_deregistration_protocol: bool,
24404        cancelled_activity: bool,
24405        draining_polls: bool,
24406        invalid_task_payload_codec: Option<InvalidTaskPayloadCodec>,
24407        workflow_completion_status: Option<&'static str>,
24408        workflow_completion_body: Option<&'static str>,
24409    }
24410
24411    impl MockWorkerServer {
24412        fn start() -> Self {
24413            Self::start_with_behavior(MockWorkerBehavior::default())
24414        }
24415
24416        fn reject_query_protocol() -> Self {
24417            Self::start_with_behavior(MockWorkerBehavior {
24418                reject_query_protocol: true,
24419                ..MockWorkerBehavior::default()
24420            })
24421        }
24422
24423        fn reject_query_completion() -> Self {
24424            Self::start_with_behavior(MockWorkerBehavior {
24425                reject_query_completion: true,
24426                ..MockWorkerBehavior::default()
24427            })
24428        }
24429
24430        fn waiting_query_worker() -> Self {
24431            Self::start_with_behavior(MockWorkerBehavior {
24432                waiting_query_worker: true,
24433                complete_named_signal: true,
24434                ..MockWorkerBehavior::default()
24435            })
24436        }
24437
24438        fn transient_worker_failures() -> Self {
24439            Self::start_with_behavior(MockWorkerBehavior {
24440                poll_failures_per_path: 1,
24441                heartbeat_failures: 1,
24442                ..MockWorkerBehavior::default()
24443            })
24444        }
24445
24446        fn consecutive_poll_failures(count: usize) -> Self {
24447            Self::start_with_behavior(MockWorkerBehavior {
24448                poll_failures_per_path: count,
24449                ..MockWorkerBehavior::default()
24450            })
24451        }
24452
24453        fn capacity_limited_activity_poll() -> Self {
24454            Self::start_with_behavior(MockWorkerBehavior {
24455                long_poll_capacity_responses_per_path: 1,
24456                ..MockWorkerBehavior::default()
24457            })
24458        }
24459
24460        fn delayed_heartbeat_worker() -> Self {
24461            Self::start_with_behavior(MockWorkerBehavior {
24462                waiting_query_worker: true,
24463                delayed_heartbeat_request: Some(2),
24464                heartbeat_response_delay: Duration::from_millis(1_500),
24465                concurrent_requests: true,
24466                cancelled_activity: true,
24467                ..MockWorkerBehavior::default()
24468            })
24469        }
24470
24471        fn heartbeat_retry_worker() -> Self {
24472            Self::start_with_behavior(MockWorkerBehavior {
24473                waiting_query_worker: true,
24474                heartbeat_failure_request: Some(2),
24475                concurrent_requests: true,
24476                ..MockWorkerBehavior::default()
24477            })
24478        }
24479
24480        fn unavailable_polls() -> Self {
24481            Self::start_with_behavior(MockWorkerBehavior {
24482                poll_failures_per_path: usize::MAX,
24483                ..MockWorkerBehavior::default()
24484            })
24485        }
24486
24487        fn unauthorized_polls() -> Self {
24488            Self::start_with_behavior(MockWorkerBehavior {
24489                unauthorized_polls: true,
24490                ..MockWorkerBehavior::default()
24491            })
24492        }
24493
24494        fn rejected_registration() -> Self {
24495            Self::start_with_behavior(MockWorkerBehavior {
24496                reject_registration: true,
24497                ..MockWorkerBehavior::default()
24498            })
24499        }
24500
24501        fn rejected_registration_protocol() -> Self {
24502            Self::start_with_behavior(MockWorkerBehavior {
24503                reject_registration_protocol: true,
24504                ..MockWorkerBehavior::default()
24505            })
24506        }
24507
24508        fn declined_registration() -> Self {
24509            Self::start_with_behavior(MockWorkerBehavior {
24510                decline_registration: true,
24511                ..MockWorkerBehavior::default()
24512            })
24513        }
24514
24515        fn rejected_deregistration() -> Self {
24516            Self::start_with_behavior(MockWorkerBehavior {
24517                reject_deregistration: true,
24518                ..MockWorkerBehavior::default()
24519            })
24520        }
24521
24522        fn rejected_deregistration_protocol() -> Self {
24523            Self::start_with_behavior(MockWorkerBehavior {
24524                reject_deregistration_protocol: true,
24525                ..MockWorkerBehavior::default()
24526            })
24527        }
24528
24529        fn unauthorized_polls_and_rejected_deregistration() -> Self {
24530            Self::start_with_behavior(MockWorkerBehavior {
24531                unauthorized_polls: true,
24532                reject_deregistration: true,
24533                ..MockWorkerBehavior::default()
24534            })
24535        }
24536
24537        fn cancelled_activity() -> Self {
24538            Self::start_with_behavior(MockWorkerBehavior {
24539                cancelled_activity: true,
24540                ..MockWorkerBehavior::default()
24541            })
24542        }
24543
24544        fn draining_polls() -> Self {
24545            Self::start_with_behavior(MockWorkerBehavior {
24546                draining_polls: true,
24547                ..MockWorkerBehavior::default()
24548            })
24549        }
24550
24551        fn invalid_task_payload_codec(codec: InvalidTaskPayloadCodec) -> Self {
24552            Self::start_with_behavior(MockWorkerBehavior {
24553                invalid_task_payload_codec: Some(codec),
24554                ..MockWorkerBehavior::default()
24555            })
24556        }
24557
24558        fn workflow_completion(status: &'static str, body: &'static str) -> Self {
24559            Self::start_with_behavior(MockWorkerBehavior {
24560                workflow_completion_status: Some(status),
24561                workflow_completion_body: Some(body),
24562                ..MockWorkerBehavior::default()
24563            })
24564        }
24565
24566        fn start_with_behavior(behavior: MockWorkerBehavior) -> Self {
24567            let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock server");
24568            listener
24569                .set_nonblocking(true)
24570                .expect("configure mock listener");
24571            let addr = listener.local_addr().expect("mock server address");
24572            let stop = Arc::new(AtomicBool::new(false));
24573            let server_stop = Arc::clone(&stop);
24574            let requests = Arc::new(Mutex::new(Vec::new()));
24575            let server_requests = Arc::clone(&requests);
24576            let thread = thread::spawn(move || {
24577                let mut request_threads = Vec::new();
24578                while !server_stop.load(Ordering::SeqCst) {
24579                    match listener.accept() {
24580                        Ok((mut stream, _)) => {
24581                            if behavior.concurrent_requests {
24582                                let requests = Arc::clone(&server_requests);
24583                                request_threads.push(thread::spawn(move || {
24584                                    handle_mock_worker_request(&mut stream, &requests, behavior)
24585                                }));
24586                            } else {
24587                                handle_mock_worker_request(&mut stream, &server_requests, behavior);
24588                            }
24589                        }
24590                        Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
24591                            let mut index = 0;
24592                            while index < request_threads.len() {
24593                                if request_threads[index].is_finished() {
24594                                    request_threads
24595                                        .swap_remove(index)
24596                                        .join()
24597                                        .expect("join mock request");
24598                                } else {
24599                                    index += 1;
24600                                }
24601                            }
24602                            thread::sleep(Duration::from_millis(5));
24603                        }
24604                        Err(_) => break,
24605                    }
24606                }
24607                for request_thread in request_threads {
24608                    request_thread.join().expect("join mock request");
24609                }
24610            });
24611
24612            Self {
24613                addr,
24614                stop,
24615                requests,
24616                thread: Some(thread),
24617            }
24618        }
24619
24620        fn base_url(&self) -> String {
24621            format!("http://{}", self.addr)
24622        }
24623
24624        fn worker_protocol_for(&self, path: &str) -> Option<String> {
24625            self.requests
24626                .lock()
24627                .expect("captured requests")
24628                .iter()
24629                .find(|request| request.path == path)
24630                .and_then(|request| request.worker_protocol.clone())
24631        }
24632
24633        fn control_protocol_for(&self, path: &str) -> Option<String> {
24634            self.requests
24635                .lock()
24636                .expect("captured requests")
24637                .iter()
24638                .find(|request| request.path == path)
24639                .and_then(|request| request.control_protocol.clone())
24640        }
24641
24642        fn method_for(&self, path: &str) -> Option<String> {
24643            self.requests
24644                .lock()
24645                .expect("captured requests")
24646                .iter()
24647                .find(|request| request.path == path)
24648                .map(|request| request.method.clone())
24649        }
24650
24651        fn authorization_for(&self, path: &str) -> Option<String> {
24652            self.requests
24653                .lock()
24654                .expect("captured requests")
24655                .iter()
24656                .find(|request| request.path == path)
24657                .and_then(|request| request.authorization.clone())
24658        }
24659
24660        fn namespace_for(&self, path: &str) -> Option<String> {
24661            self.requests
24662                .lock()
24663                .expect("captured requests")
24664                .iter()
24665                .find(|request| request.path == path)
24666                .and_then(|request| request.namespace.clone())
24667        }
24668
24669        fn request_count(&self, path: &str) -> usize {
24670            self.requests
24671                .lock()
24672                .expect("captured requests")
24673                .iter()
24674                .filter(|request| request.path == path)
24675                .count()
24676        }
24677
24678        fn captured_paths(&self) -> Vec<String> {
24679            self.requests
24680                .lock()
24681                .expect("captured requests")
24682                .iter()
24683                .map(|request| request.path.clone())
24684                .collect()
24685        }
24686
24687        fn request_times(&self, path: &str) -> Vec<Instant> {
24688            self.requests
24689                .lock()
24690                .expect("captured requests")
24691                .iter()
24692                .filter(|request| request.path == path)
24693                .map(|request| request.received_at)
24694                .collect()
24695        }
24696
24697        fn request_body(&self, path: &str) -> Value {
24698            let requests = self.requests.lock().expect("captured requests");
24699            let body = &requests
24700                .iter()
24701                .find(|request| request.path == path)
24702                .unwrap_or_else(|| panic!("missing request for {path}"))
24703                .body;
24704            serde_json::from_str(body).unwrap_or_else(|error| {
24705                panic!("invalid JSON request body for {path}: {error}: {body:?}")
24706            })
24707        }
24708
24709        fn request_bodies(&self, path: &str) -> Vec<Value> {
24710            self.requests
24711                .lock()
24712                .expect("captured requests")
24713                .iter()
24714                .filter(|request| request.path == path)
24715                .map(|request| {
24716                    serde_json::from_str(&request.body).unwrap_or_else(|error| {
24717                        panic!(
24718                            "invalid JSON request body for {path}: {error}: {:?}",
24719                            request.body
24720                        )
24721                    })
24722                })
24723                .collect()
24724        }
24725    }
24726
24727    impl Drop for MockWorkerServer {
24728        fn drop(&mut self) {
24729            self.stop.store(true, Ordering::SeqCst);
24730            let _ = TcpStream::connect(self.addr);
24731
24732            if let Some(thread) = self.thread.take() {
24733                thread.join().expect("join mock server");
24734            }
24735        }
24736    }
24737
24738    fn handle_mock_worker_request(
24739        stream: &mut TcpStream,
24740        requests: &Arc<Mutex<Vec<CapturedRequest>>>,
24741        behavior: MockWorkerBehavior,
24742    ) {
24743        let _ = stream.set_read_timeout(Some(Duration::from_millis(200)));
24744        let mut buffer = [0_u8; 8192];
24745        let mut request = Vec::new();
24746
24747        loop {
24748            match stream.read(&mut buffer) {
24749                Ok(0) => break,
24750                Ok(read) => {
24751                    request.extend_from_slice(&buffer[..read]);
24752                    if mock_request_is_complete(&request) {
24753                        break;
24754                    }
24755                }
24756                Err(error)
24757                    if matches!(
24758                        error.kind(),
24759                        std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
24760                    ) =>
24761                {
24762                    break;
24763                }
24764                Err(_) => return,
24765            }
24766        }
24767
24768        let request = String::from_utf8_lossy(&request);
24769        let body = request
24770            .split_once("\r\n\r\n")
24771            .map(|(_, body)| body)
24772            .unwrap_or_default();
24773        let path = request
24774            .lines()
24775            .next()
24776            .and_then(|line| line.split_whitespace().nth(1))
24777            .unwrap_or_default();
24778        let method = request
24779            .lines()
24780            .next()
24781            .and_then(|line| line.split_whitespace().next())
24782            .unwrap_or_default();
24783        let authorization = request.lines().find_map(|line| {
24784            let (name, value) = line.split_once(':')?;
24785            name.eq_ignore_ascii_case("Authorization")
24786                .then(|| value.trim().to_string())
24787        });
24788        let namespace = request.lines().find_map(|line| {
24789            let (name, value) = line.split_once(':')?;
24790            name.eq_ignore_ascii_case("X-Namespace")
24791                .then(|| value.trim().to_string())
24792        });
24793        let worker_protocol = request.lines().find_map(|line| {
24794            let (name, value) = line.split_once(':')?;
24795            name.eq_ignore_ascii_case("X-Durable-Workflow-Protocol-Version")
24796                .then(|| value.trim().to_string())
24797        });
24798        let control_protocol = request.lines().find_map(|line| {
24799            let (name, value) = line.split_once(':')?;
24800            name.eq_ignore_ascii_case("X-Durable-Workflow-Control-Plane-Version")
24801                .then(|| value.trim().to_string())
24802        });
24803        let request_number = {
24804            let mut requests = requests.lock().expect("captured requests");
24805            requests.push(CapturedRequest {
24806                headers: request
24807                    .split_once("\r\n\r\n")
24808                    .map_or("", |(headers, _)| headers)
24809                    .to_owned(),
24810                method: method.to_string(),
24811                path: path.to_string(),
24812                authorization,
24813                namespace,
24814                worker_protocol: worker_protocol.clone(),
24815                control_protocol,
24816                body: body.to_string(),
24817                received_at: Instant::now(),
24818            });
24819            requests
24820                .iter()
24821                .filter(|request| request.path == path)
24822                .count()
24823        };
24824
24825        if let Some(response) = behavior
24826            .request_override
24827            .and_then(|handler| handler(path, body, request_number))
24828        {
24829            write_mock_response(stream, response.0, &response.1);
24830            return;
24831        }
24832        if let Some(response) = behavior.response_override.and_then(|handler| handler(path)) {
24833            write_mock_response(stream, response.0, &response.1);
24834            return;
24835        }
24836        if path.ends_with("/poll") && request_number <= behavior.poll_failures_per_path {
24837            return;
24838        }
24839        let pressure_path = behavior
24840            .storage_path
24841            .is_some_and(|part| path.contains(part));
24842        let prior_failures = if path.ends_with("/poll") {
24843            behavior.poll_failures_per_path
24844        } else {
24845            0
24846        };
24847        if pressure_path
24848            && request_number.saturating_sub(prior_failures) <= behavior.storage_refusals
24849        {
24850            let request_body: Value = serde_json::from_str(body).unwrap_or(Value::Null);
24851            let poll_id = path
24852                .ends_with("/poll")
24853                .then(|| request_body["poll_request_id"].as_str().unwrap_or(""));
24854            let mut refusal = storage_refusal(
24855                poll_id,
24856                behavior.storage_unavailable,
24857                behavior.storage_mid_poll,
24858            );
24859            if behavior.storage_wrong_poll_id {
24860                refusal["poll_request_id"] = json!("wrong-poll");
24861            }
24862            write_mock_response(stream, "503 Service Unavailable", &refusal.to_string());
24863            return;
24864        }
24865        if path.contains("/storage-task/")
24866            || path.contains("/storage-activity/")
24867            || path.contains("/storage-query/")
24868        {
24869            write_mock_response(stream, "200 OK", "{}");
24870            return;
24871        }
24872        if behavior.storage_query && path == "/api/worker/query-tasks/poll" && request_number == 1 {
24873            write_mock_response(stream, "200 OK", &json!({"task":{
24874                "query_task_id":"storage-query", "query_task_attempt":7, "workflow_type":"storage.workflow",
24875                "query_name":"state", "workflow_id":"workflow", "run_id":"run", "payload_codec":"avro",
24876                "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24877                "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24878                "history_events":[], "run_status":"waiting", "lease_owner":"storage-worker"
24879            }}).to_string());
24880            return;
24881        }
24882        if behavior.storage_activity
24883            && path == "/api/worker/activity-tasks/poll"
24884            && request_number == 1
24885        {
24886            write_mock_response(stream, "200 OK", &json!({"task":{
24887                "task_id":"storage-activity", "activity_attempt_id":"storage-attempt", "activity_type":"storage.activity",
24888                "payload_codec":"avro", "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC).unwrap(),
24889                "attempt_number":7, "lease_owner":"storage-worker"
24890            }}).to_string());
24891            return;
24892        }
24893
24894        if path == "/api/worker/register" {
24895            if behavior.reject_registration_protocol {
24896                write_mock_response(
24897                    stream,
24898                    "400 Bad Request",
24899                    r#"{"reason":"unsupported_protocol_version","message":"condition-wait occurrence identity requires worker protocol 1.17","supported_version":"1.16","requested_version":"1.17"}"#,
24900                );
24901                return;
24902            }
24903            if behavior.reject_registration {
24904                write_mock_response(
24905                    stream,
24906                    "503 Service Unavailable",
24907                    r#"{"reason":"registration_unavailable","message":"registration failed"}"#,
24908                );
24909                return;
24910            }
24911        }
24912
24913        if path.starts_with("/api/worker/registrations/") {
24914            if behavior.reject_deregistration_protocol {
24915                write_mock_response(
24916                    stream,
24917                    "400 Bad Request",
24918                    r#"{"reason":"unsupported_protocol_version","message":"unsupported worker protocol","supported_version":"1.17","requested_version":"1.19"}"#,
24919                );
24920            } else if behavior.reject_deregistration {
24921                write_mock_response(
24922                    stream,
24923                    "403 Forbidden",
24924                    r#"{"reason":"authorization_failed","message":"worker cannot deregister"}"#,
24925                );
24926            } else {
24927                write_mock_response(
24928                    stream,
24929                    "200 OK",
24930                    r#"{"worker_id":"deregistered-worker","outcome":"deregistered","recovered_workflow_task_count":2}"#,
24931                );
24932            }
24933            return;
24934        }
24935
24936        let is_poll = matches!(
24937            path,
24938            "/api/worker/workflow-tasks/poll"
24939                | "/api/worker/activity-tasks/poll"
24940                | "/api/worker/query-tasks/poll"
24941        );
24942        if is_poll && request_number <= behavior.long_poll_capacity_responses_per_path {
24943            write_mock_response(
24944                stream,
24945                "429 Too Many Requests",
24946                r#"{"task":null,"poll_status":"long_poll_capacity_exhausted","reason":"long_poll_capacity_exhausted","retryable":true,"retry_after_seconds":1}"#,
24947            );
24948            return;
24949        }
24950        if is_poll && request_number <= behavior.poll_failures_per_path {
24951            return;
24952        }
24953        if path == "/api/worker/heartbeat" && request_number <= behavior.heartbeat_failures {
24954            return;
24955        }
24956        if path == "/api/worker/heartbeat"
24957            && behavior.heartbeat_failure_request == Some(request_number)
24958        {
24959            return;
24960        }
24961        if path == "/api/worker/heartbeat"
24962            && behavior.delayed_heartbeat_request == Some(request_number)
24963        {
24964            thread::sleep(behavior.heartbeat_response_delay);
24965        }
24966        if behavior.unauthorized_polls && is_poll {
24967            write_mock_response(
24968                stream,
24969                "401 Unauthorized",
24970                r#"{"reason":"authentication_failed","message":"invalid worker token"}"#,
24971            );
24972            return;
24973        }
24974        if behavior.draining_polls && is_poll {
24975            write_mock_response(
24976                stream,
24977                "409 Conflict",
24978                r#"{"task":null,"poll_status":"draining","reason":"worker_draining","worker_status":"draining","drain_intent":"draining"}"#,
24979            );
24980            return;
24981        }
24982
24983        if let Some(codec_case) = behavior.invalid_task_payload_codec {
24984            if is_poll && request_number == 1 {
24985                let mut task = match path {
24986                    "/api/worker/workflow-tasks/poll" => json!({
24987                        "task_id": "codec-workflow",
24988                        "workflow_type": "codec.workflow",
24989                        "payload_codec": DEFAULT_CODEC,
24990                        "workflow_task_attempt": 1,
24991                        "lease_owner": "codec-worker"
24992                    }),
24993                    "/api/worker/activity-tasks/poll" => json!({
24994                        "task_id": "codec-activity",
24995                        "activity_attempt_id": "codec-activity-attempt",
24996                        "activity_type": "codec.activity",
24997                        "payload_codec": DEFAULT_CODEC,
24998                        "attempt_number": 1,
24999                        "lease_owner": "codec-worker"
25000                    }),
25001                    "/api/worker/query-tasks/poll" => json!({
25002                        "query_task_id": "codec-query",
25003                        "query_task_attempt": 1,
25004                        "workflow_type": "codec.workflow",
25005                        "query_name": "known",
25006                        "payload_codec": DEFAULT_CODEC,
25007                        "lease_owner": "codec-worker"
25008                    }),
25009                    _ => unreachable!("is_poll limits task codec probe paths"),
25010                };
25011                codec_case.apply(&mut task);
25012                write_mock_response(stream, "200 OK", &json!({"task": task}).to_string());
25013                return;
25014            }
25015
25016            if matches!(
25017                path,
25018                "/api/worker/workflow-tasks/codec-workflow/fail"
25019                    | "/api/worker/activity-tasks/codec-activity/fail"
25020                    | "/api/worker/query-tasks/codec-query/fail"
25021            ) {
25022                write_mock_response(stream, "200 OK", r#"{"outcome":"failed"}"#);
25023                return;
25024            }
25025        }
25026
25027        if behavior.reject_query_protocol && path.starts_with("/api/worker/query-tasks/") {
25028            let requested_version = worker_protocol.as_deref().unwrap_or("missing");
25029            let body = format!(
25030                r#"{{"reason":"unsupported_protocol_version","message":"Query tasks require worker protocol 1.8 or newer.","supported_version":"1.7","requested_version":"{requested_version}"}}"#
25031            );
25032            write_mock_response(stream, "400 Bad Request", &body);
25033            return;
25034        }
25035
25036        if behavior.reject_query_completion && path == "/api/worker/query-tasks/query-late/complete"
25037        {
25038            write_mock_response(
25039                stream,
25040                "409 Conflict",
25041                r#"{"reason":"query_task_timed_out","message":"query task timed out before completion"}"#,
25042            );
25043            return;
25044        }
25045
25046        if behavior.workflow_completion_status.is_some()
25047            && path == "/api/worker/workflow-tasks/poll"
25048            && request_number == 1
25049        {
25050            write_mock_response(
25051                stream,
25052                "200 OK",
25053                r#"{"task":{"task_id":"workflow-timeout-task","workflow_id":"reused-workflow-id","run_id":"run-selected-timeout","workflow_type":"timeout.workflow","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"workflow_task_attempt":3,"lease_owner":"timeout-worker"}}"#,
25054            );
25055            return;
25056        }
25057
25058        if path == "/api/worker/workflow-tasks/workflow-timeout-task/complete" {
25059            if let (Some(status), Some(body)) = (
25060                behavior.workflow_completion_status,
25061                behavior.workflow_completion_body,
25062            ) {
25063                write_mock_response(stream, status, body);
25064                return;
25065            }
25066        }
25067
25068        if behavior.waiting_query_worker {
25069            if behavior.complete_named_signal
25070                && path == "/api/worker/workflow-tasks/poll"
25071                && request_number == 1
25072            {
25073                let body = json!({
25074                    "task": {
25075                        "task_id": "snapshot-open",
25076                        "workflow_id": "snapshot-1",
25077                        "run_id": "snapshot-run-1",
25078                        "workflow_type": "snapshot",
25079                        "payload_codec": DEFAULT_CODEC,
25080                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25081                            .expect("Avro workflow arguments"),
25082                        "history_events": [],
25083                        "workflow_task_attempt": 1,
25084                        "lease_owner": "rust-snapshot-worker"
25085                    }
25086                })
25087                .to_string();
25088                write_mock_response(stream, "200 OK", &body);
25089                return;
25090            }
25091
25092            let signal_request = request_number - usize::from(behavior.complete_named_signal);
25093            let signal_request_limit = 2 + usize::from(behavior.complete_named_signal);
25094            if path == "/api/worker/workflow-tasks/poll"
25095                && signal_request >= 1
25096                && signal_request <= signal_request_limit
25097            {
25098                let finish = behavior.complete_named_signal && signal_request == 3;
25099                let amounts = if signal_request == 1 {
25100                    vec![3]
25101                } else {
25102                    vec![3, 5]
25103                };
25104                let task_id = if signal_request == 1 {
25105                    "snapshot-wait-3"
25106                } else if finish {
25107                    "snapshot-finish"
25108                } else {
25109                    "snapshot-wait-5"
25110                };
25111                let mut history_events = std::iter::once(json!({
25112                    "event_type": "SignalWaitOpened",
25113                    "payload": {"sequence": 1, "signal_name": "finish"}
25114                }))
25115                .chain(amounts.iter().enumerate().map(|(index, amount)| {
25116                    json!({
25117                        "event_type": "SignalReceived",
25118                        "payload": {
25119                            "signal_id": format!("increment-{amount}"),
25120                            "signal_name": "increment",
25121                            "workflow_sequence": index + 2,
25122                            "payload_codec": DEFAULT_CODEC,
25123                            "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25124                                .expect("Avro signal envelope")
25125                        }
25126                    })
25127                }))
25128                .collect::<Vec<_>>();
25129                let (resume_id, resume_name, resume_arguments) = if finish {
25130                    history_events.push(json!({
25131                        "event_type": "SignalReceived",
25132                        "payload": {
25133                            "signal_id": "finish",
25134                            "signal_name": "finish",
25135                            "workflow_sequence": 4,
25136                            "payload_codec": DEFAULT_CODEC,
25137                            "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25138                                .expect("Avro finish signal envelope")
25139                        }
25140                    }));
25141                    (
25142                        "finish".to_string(),
25143                        "finish".to_string(),
25144                        encode_value_envelope(&json!([]), DEFAULT_CODEC)
25145                            .expect("Avro finish resume signal"),
25146                    )
25147                } else {
25148                    let amount = amounts.last().expect("amount");
25149                    (
25150                        format!("increment-{amount}"),
25151                        "increment".to_string(),
25152                        encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25153                            .expect("Avro increment resume signal"),
25154                    )
25155                };
25156                let body = json!({
25157                    "task": {
25158                        "task_id": task_id,
25159                        "workflow_id": "snapshot-1",
25160                        "run_id": "snapshot-run-1",
25161                        "workflow_type": "snapshot",
25162                        "payload_codec": DEFAULT_CODEC,
25163                        "arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25164                            .expect("Avro workflow arguments"),
25165                        "history_events": history_events,
25166                        "workflow_task_attempt": 1,
25167                        "workflow_signal_id": resume_id,
25168                        "signal_name": resume_name,
25169                        "signal_arguments": resume_arguments,
25170                        "lease_owner": "rust-snapshot-worker"
25171                    }
25172                })
25173                .to_string();
25174                write_mock_response(stream, "200 OK", &body);
25175                return;
25176            }
25177
25178            if path == "/api/worker/query-tasks/poll" && request_number == 1 {
25179                let history_events = [3, 5]
25180                    .into_iter()
25181                    .enumerate()
25182                    .map(|(index, amount)| {
25183                        json!({
25184                            "event_type": "SignalReceived",
25185                            "payload": {
25186                                "signal_id": format!("increment-{amount}"),
25187                                "signal_name": "increment",
25188                                "workflow_sequence": index + 2,
25189                                "payload_codec": DEFAULT_CODEC,
25190                                "arguments": encode_value_envelope(&json!([amount]), DEFAULT_CODEC)
25191                                    .expect("Avro query signal envelope")
25192                            }
25193                        })
25194                    })
25195                    .collect::<Vec<_>>();
25196                let body = json!({
25197                    "task": {
25198                        "query_task_id": "snapshot-current",
25199                        "query_task_attempt": 1,
25200                        "lease_owner": "rust-snapshot-worker",
25201                        "workflow_id": "snapshot-1",
25202                        "run_id": "snapshot-run-1",
25203                        "workflow_type": "snapshot",
25204                        "query_name": "current",
25205                        "payload_codec": DEFAULT_CODEC,
25206                        "workflow_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25207                            .expect("Avro workflow arguments"),
25208                        "query_arguments": encode_value_envelope(&json!([]), DEFAULT_CODEC)
25209                            .expect("Avro query arguments"),
25210                        "history_events": history_events,
25211                        "run_status": "waiting"
25212                    }
25213                })
25214                .to_string();
25215                write_mock_response(stream, "200 OK", &body);
25216                return;
25217            }
25218
25219            if path == "/api/worker/workflow-tasks/snapshot-wait-3/fail"
25220                || path == "/api/worker/workflow-tasks/snapshot-wait-5/fail"
25221            {
25222                write_mock_response(
25223                    stream,
25224                    "200 OK",
25225                    r#"{"outcome":"waiting_for_history","recorded":true}"#,
25226                );
25227                return;
25228            }
25229
25230            if path == "/api/worker/workflow-tasks/snapshot-open/complete" {
25231                write_mock_response(stream, "200 OK", r#"{"outcome":"waiting","recorded":true}"#);
25232                return;
25233            }
25234
25235            if path == "/api/worker/workflow-tasks/snapshot-finish/complete" {
25236                write_mock_response(
25237                    stream,
25238                    "200 OK",
25239                    r#"{"outcome":"completed","run_status":"completed","recorded":true}"#,
25240                );
25241                return;
25242            }
25243
25244            if path == "/api/worker/query-tasks/snapshot-current/complete" {
25245                write_mock_response(stream, "200 OK", r#"{"outcome":"completed"}"#);
25246                return;
25247            }
25248        }
25249
25250        if matches!(
25251            path,
25252            "/api/workflows/typed-1/query/inspect" | "/api/workflows/typed-1/update/replace"
25253        ) {
25254            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25255                .expect("typed mock result");
25256            let body = json!({
25257                "result": typed_fidelity_probe().into_json().expect("result projection"),
25258                "result_envelope": result,
25259            })
25260            .to_string();
25261            write_mock_response(stream, "200 OK", &body);
25262            return;
25263        }
25264
25265        if path == "/api/workflows/typed-1" {
25266            let result = encode_typed_envelope(&typed_fidelity_probe(), DEFAULT_CODEC)
25267                .expect("typed mock result");
25268            let body = json!({
25269                "workflow_id": "typed-1",
25270                "run_id": "run-typed-1",
25271                "workflow_type": "typed.echo",
25272                "status": "completed",
25273                "output": typed_fidelity_probe().into_json().expect("output projection"),
25274                "output_envelope": result,
25275            })
25276            .to_string();
25277            write_mock_response(stream, "200 OK", &body);
25278            return;
25279        }
25280
25281        let (status, body) = match path {
25282            "/api/cluster/info" => ("200 OK", r#"{"limits":{"max_payload_bytes":2097152}}"#),
25283            "/api/health" => ("200 OK", r#"{"status":"ok"}"#),
25284            "/api/workflows" => (
25285                "201 Created",
25286                r#"{"workflow_id":"wf-start-options","run_id":"run-start-options","workflow_type":"rust.timeout"}"#,
25287            ),
25288            "/api/worker/register" if behavior.decline_registration => (
25289                "200 OK",
25290                r#"{"worker_id":"declined-worker","registered":false}"#,
25291            ),
25292            "/api/worker/register" if behavior.waiting_query_worker => (
25293                "200 OK",
25294                r#"{"worker_id":"rust-snapshot-worker","registered":true,"heartbeat_interval_seconds":1}"#,
25295            ),
25296            "/api/worker/register" => (
25297                "200 OK",
25298                r#"{"worker_id":"mock-worker","registered":true,"heartbeat_interval_seconds":3600}"#,
25299            ),
25300            "/api/worker/heartbeat" => ("200 OK", "{}"),
25301            "/api/worker/activity-tasks/poll"
25302                if behavior.cancelled_activity && request_number == 1 =>
25303            {
25304                (
25305                    "200 OK",
25306                    r#"{"task":{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","activity_type":"cancel-aware","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"rust-cancel-worker"}}"#,
25307                )
25308            }
25309            "/api/worker/activity-tasks/poll"
25310                if behavior.long_poll_capacity_responses_per_path > 0
25311                    && request_number
25312                        == behavior
25313                            .long_poll_capacity_responses_per_path
25314                            .saturating_add(1) =>
25315            {
25316                (
25317                    "200 OK",
25318                    r#"{"task":{"task_id":"capacity-activity","activity_attempt_id":"capacity-attempt","activity_type":"capacity.activity","payload_codec":"avro","arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"attempt_number":1,"lease_owner":"capacity-worker"}}"#,
25319                )
25320            }
25321            "/api/worker/activity-tasks/poll" | "/api/worker/workflow-tasks/poll" => {
25322                ("200 OK", r#"{"task":null}"#)
25323            }
25324            "/api/worker/query-tasks/poll"
25325                if behavior.reject_query_completion && request_number == 1 =>
25326            {
25327                (
25328                    "200 OK",
25329                    r#"{"task":{"query_task_id":"query-late","query_task_attempt":1,"lease_owner":"late-worker","workflow_id":"counter-late","run_id":"run-late","workflow_type":"counter","query_name":"current","payload_codec":"avro","workflow_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"query_arguments":{"codec":"avro","blob":"wwHioz3/VYAiNwwA"},"history_events":[],"run_status":"running"}}"#,
25330                )
25331            }
25332            "/api/worker/query-tasks/poll" => ("200 OK", r#"{"task":null}"#),
25333            "/api/worker/query-tasks/query-capture/complete"
25334            | "/api/worker/query-tasks/query-capture/fail" => ("200 OK", "{}"),
25335            "/api/worker/activity-tasks/activity-cancel/heartbeat" => (
25336                "200 OK",
25337                r#"{"activity_attempt_id":"attempt-cancel","cancel_requested":true,"can_continue":false,"reason":"run_cancelled","run_closed_reason":"cancelled","heartbeat_recorded":false}"#,
25338            ),
25339            "/api/worker/activity-tasks/activity-cancel/complete" => (
25340                "409 Conflict",
25341                r#"{"task_id":"activity-cancel","activity_attempt_id":"attempt-cancel","reason":"run_cancelled","cancel_requested":true,"can_continue":false,"run_closed_reason":"cancelled"}"#,
25342            ),
25343            "/api/worker/activity-tasks/activity-typed/complete"
25344            | "/api/worker/activity-tasks/activity-typed/fail"
25345            | "/api/worker/activity-tasks/capacity-activity/complete"
25346            | "/api/workflows/typed-1/signal/changed" => ("200 OK", "{}"),
25347            "/api/workflows/counter-1/query/current" => (
25348                "200 OK",
25349                r#"{"workflow_id":"counter-1","query_name":"current","result":{"count":8},"result_envelope":{"codec":"avro","blob":"wwHioz3/VYAiNw4CCmNvdW50BBAA"}}"#,
25350            ),
25351            "/api/workflows/counter-1/query/missing" => (
25352                "404 Not Found",
25353                r#"{"workflow_id":"counter-1","query_name":"missing","reason":"rejected_unknown_query","message":"unknown query"}"#,
25354            ),
25355            "/api/workflows/wf-lifecycle/cancel" => (
25356                "200 OK",
25357                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","reason":"cleanup requested","command_status":"accepted"}"#,
25358            ),
25359            "/api/workflows/wf-lifecycle/terminate" => (
25360                "200 OK",
25361                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","reason":"forced stop","command_status":"accepted"}"#,
25362            ),
25363            "/api/workflows/wf-lifecycle/runs/run-current/cancel" => (
25364                "200 OK",
25365                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"cancelled","command_status":"accepted"}"#,
25366            ),
25367            "/api/workflows/wf-lifecycle/runs/run-current/terminate" => (
25368                "200 OK",
25369                r#"{"workflow_id":"wf-lifecycle","run_id":"run-current","outcome":"terminated","command_status":"accepted"}"#,
25370            ),
25371            "/api/workflows/wf-lifecycle/runs/run-stale/cancel"
25372            | "/api/workflows/wf-lifecycle/runs/run-stale/terminate" => (
25373                "409 Conflict",
25374                r#"{"workflow_id":"wf-lifecycle","run_id":"run-stale","reason":"historical_run_command_rejected","target_scope":"run","message":"Commands cannot target historical runs."}"#,
25375            ),
25376            "/api/workflows/wf-failed" | "/api/workflows/wf-failed/runs/run-failed" => (
25377                "200 OK",
25378                r#"{"workflow_id":"wf-failed","run_id":"run-failed","status":"failed","closed_reason":"failed","error":"payment failed","failure":{"message":"payment failed","failure_category":"application","exception_type":"PaymentError","exception_class":"billing::PaymentError","non_retryable":true,"exception":{"type":"PaymentError","class":"billing::PaymentError","message":"payment failed"},"failures":[{"id":"failure-17","failure_category":"application"}]}}"#,
25379            ),
25380            "/api/workflows/wf-cancelled" => (
25381                "200 OK",
25382                r#"{"workflow_id":"wf-cancelled","run_id":"run-cancelled","status":"cancelled","closed_reason":"cancelled","reason":"cleanup requested"}"#,
25383            ),
25384            "/api/workflows/wf-terminated" => (
25385                "200 OK",
25386                r#"{"workflow_id":"wf-terminated","run_id":"run-terminated","status":"terminated","closed_reason":"terminated","reason":"forced stop"}"#,
25387            ),
25388            "/api/workflows/wf-timed-out" => (
25389                "200 OK",
25390                r#"{"workflow_id":"wf-timed-out","run_id":"run-timed-out","status":"failed","closed_reason":"timed_out","reason":"run_timeout"}"#,
25391            ),
25392            "/api/workflows/wf-waiting" | "/api/workflows/wf-waiting/runs/run-waiting" => (
25393                "200 OK",
25394                r#"{"workflow_id":"wf-waiting","run_id":"run-waiting","status":"waiting"}"#,
25395            ),
25396            "/api/workflows/wf-selected" => (
25397                "200 OK",
25398                r#"{"workflow_id":"wf-selected","run_id":"run-current","status":"completed","output":"current run output"}"#,
25399            ),
25400            "/api/workflows/wf-selected/runs/run-selected" => (
25401                "200 OK",
25402                r#"{"workflow_id":"wf-selected","run_id":"run-selected","status":"cancelled","closed_reason":"cancelled","reason":"selected run cancelled"}"#,
25403            ),
25404            _ => ("404 Not Found", r#"{"message":"not found"}"#),
25405        };
25406        write_mock_response(stream, status, body);
25407    }
25408
25409    fn mock_request_is_complete(request: &[u8]) -> bool {
25410        let Some(header_end) = request
25411            .windows(4)
25412            .position(|window| window == b"\r\n\r\n")
25413            .map(|position| position + 4)
25414        else {
25415            return false;
25416        };
25417        let headers = String::from_utf8_lossy(&request[..header_end]);
25418        let content_length = headers.lines().find_map(|line| {
25419            let (name, value) = line.split_once(':')?;
25420            name.eq_ignore_ascii_case("content-length")
25421                .then(|| value.trim().parse::<usize>().ok())
25422                .flatten()
25423        });
25424
25425        request.len() >= header_end + content_length.unwrap_or(0)
25426    }
25427
25428    fn write_mock_response(stream: &mut TcpStream, status: &str, body: &str) {
25429        let response = format!(
25430            "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
25431            body.len()
25432        );
25433
25434        let _ = stream.write_all(response.as_bytes());
25435        let _ = stream.flush();
25436    }
25437}